diff --git a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml index ee756236f..226830c31 100644 --- a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml +++ b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-AL2023-int-test-workflow.yml b/.github/workflows/arm-AL2023-int-test-workflow.yml index 39986ebca..a65cf941f 100644 --- a/.github/workflows/arm-AL2023-int-test-workflow.yml +++ b/.github/workflows/arm-AL2023-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-al2023 diff --git a/.github/workflows/arm-RHEL-build-test-push-workflow.yml b/.github/workflows/arm-RHEL-build-test-push-workflow.yml index 07848683c..f86a5e632 100644 --- a/.github/workflows/arm-RHEL-build-test-push-workflow.yml +++ b/.github/workflows/arm-RHEL-build-test-push-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-RHEL-int-test-workflow.yml b/.github/workflows/arm-RHEL-int-test-workflow.yml index 83c7e8fb9..d2a127ca8 100644 --- a/.github/workflows/arm-RHEL-int-test-workflow.yml +++ b/.github/workflows/arm-RHEL-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml index a22bd10c4..5c6179c20 100644 --- a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml +++ b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-Ubuntu-int-test-workflow.yml b/.github/workflows/arm-Ubuntu-int-test-workflow.yml index b3f133abe..8d138f5bb 100644 --- a/.github/workflows/arm-Ubuntu-int-test-workflow.yml +++ b/.github/workflows/arm-Ubuntu-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-ubuntu diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index 01e655adb..dda66569a 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -200,7 +200,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-build-test-push-workflow.yml b/.github/workflows/distroless-build-test-push-workflow.yml index 8aa6172f7..63d9bee33 100644 --- a/.github/workflows/distroless-build-test-push-workflow.yml +++ b/.github/workflows/distroless-build-test-push-workflow.yml @@ -195,7 +195,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-int-test-workflow.yml b/.github/workflows/distroless-int-test-workflow.yml index 01a024247..5f16a8472 100644 --- a/.github/workflows/distroless-int-test-workflow.yml +++ b/.github/workflows/distroless-int-test-workflow.yml @@ -73,7 +73,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-distroless diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 001a34cee..c545b69eb 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -70,7 +70,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml index 96629316e..0e3e5d357 100644 --- a/.github/workflows/manual-int-test-workflow.yml +++ b/.github/workflows/manual-int-test-workflow.yml @@ -29,7 +29,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/namespace-scope-int-workflow.yml b/.github/workflows/namespace-scope-int-workflow.yml index fc3d3554f..640f74dc6 100644 --- a/.github/workflows/namespace-scope-int-workflow.yml +++ b/.github/workflows/namespace-scope-int-workflow.yml @@ -25,7 +25,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 3f886dd9a..184927f04 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -66,7 +66,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/test/appframework_aws/c3/appframework_aws_suite_test.go b/test/appframework_aws/c3/appframework_aws_suite_test.go index 6468b018a..8df2aca24 100644 --- a/test/appframework_aws/c3/appframework_aws_suite_test.go +++ b/test/appframework_aws/c3/appframework_aws_suite_test.go @@ -22,19 +22,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -42,9 +32,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -55,9 +42,7 @@ var ( func TestBasic(t *testing.T) { RegisterFailHandler(Fail) - // Find and load the .env file from the current directory upwards - err := loadEnvFile() - Expect(err).ToNot(HaveOccurred(), "Error loading .env file") + Expect(testenv.LoadEnvFile()).ToNot(HaveOccurred(), "Error loading .env file") sc, _ := GinkgoConfiguration() sc.Timeout = 240 * time.Minute @@ -65,77 +50,10 @@ func TestBasic(t *testing.T) { RunSpecs(t, "Running "+testSuiteName, sc) } -//func TestMain(m *testing.M) { -// Run the tests -// os.Exit(m.Run()) -//} - -// loadEnvFile traverses up the directory tree to find a .env file -func loadEnvFile() error { - // Get the current working directory - dir, err := os.Getwd() - if err != nil { - return err - } - - // Traverse up the directory tree - for { - // Check if .env file exists in the current directory - envFile := filepath.Join(dir, ".env") - if _, err := os.Stat(envFile); err == nil { - // .env file found, load it - return godotenv.Load(envFile) - } - - // Move up to the parent directory - parentDir := filepath.Dir(dir) - if parentDir == dir { - // Reached the root directory - return nil - } - dir = parentDir - } -} - var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework_aws/c3/appframework_aws_test.go index 857ba1742..6381ffeca 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework_aws/c3/appframework_aws_test.go @@ -25,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -53,44 +52,14 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") testenv.SpecifiedTestTimeout = 4000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -196,20 +165,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -277,19 +238,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -417,20 +371,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -486,19 +432,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -596,11 +535,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -693,12 +628,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas), "main", 2000) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -877,11 +807,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -926,11 +852,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1066,11 +988,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1130,11 +1048,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1273,11 +1187,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1353,11 +1263,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1413,7 +1319,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1451,11 +1357,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1578,11 +1480,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1640,11 +1538,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1790,11 +1684,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1839,11 +1729,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2015,11 +1901,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2091,11 +1973,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2187,7 +2065,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2316,7 +2194,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2372,11 +2250,7 @@ var _ = Describe("c3appfw test", func() { // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) @@ -2419,7 +2293,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2461,11 +2335,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2513,7 +2383,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2555,11 +2425,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2640,11 +2506,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2726,7 +2588,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2790,11 +2652,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2843,7 +2701,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2884,11 +2742,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2932,7 +2786,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2977,11 +2831,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3061,11 +2911,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3156,14 +3002,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3242,11 +3088,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3271,7 +3113,7 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework_aws/c3/manager_appframework_test.go index a49fd3eb4..384cf61ec 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework_aws/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,44 +50,13 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -194,20 +162,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -275,19 +235,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -558,20 +511,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -627,19 +572,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -737,11 +675,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -838,12 +772,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas), "main", 2000) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -1026,11 +955,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1075,11 +1000,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1215,11 +1136,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1279,11 +1196,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1422,11 +1335,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1502,11 +1411,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1562,7 +1467,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1600,11 +1505,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1727,11 +1628,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1789,11 +1686,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1939,11 +1832,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1988,11 +1877,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2164,11 +2049,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2240,11 +2121,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2336,7 +2213,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2465,7 +2342,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2568,7 +2445,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2610,11 +2487,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2662,7 +2535,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2704,11 +2577,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2789,11 +2658,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2875,7 +2740,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2939,11 +2804,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2992,7 +2853,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -3033,11 +2894,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3081,7 +2938,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -3126,11 +2983,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3210,11 +3063,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3305,14 +3154,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3391,11 +3240,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3420,7 +3265,7 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster diff --git a/test/appframework_aws/c3/test_helpers_test.go b/test/appframework_aws/c3/test_helpers_test.go new file mode 100644 index 000000000..379908bce --- /dev/null +++ b/test/appframework_aws/c3/test_helpers_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package c3appfw + +import ( + "context" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifyC3ClusterReady verifies SHC is ready and single-site indexers are ready. +func verifyC3ClusterReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment) { + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// verifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func verifyMCVersionChangedAndReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) { + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} diff --git a/test/appframework_aws/m4/appframework_aws_suite_test.go b/test/appframework_aws/m4/appframework_aws_suite_test.go index aa21c7084..419a76b06 100644 --- a/test/appframework_aws/m4/appframework_aws_suite_test.go +++ b/test/appframework_aws/m4/appframework_aws_suite_test.go @@ -25,15 +25,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +32,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,7 +40,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() @@ -62,44 +49,9 @@ func TestBasic(t *testing.T) { } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework_aws/m4/appframework_aws_test.go index fca638acb..45c71f82c 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework_aws/m4/appframework_aws_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -53,16 +52,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,27 +61,9 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { @@ -192,14 +164,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -273,14 +238,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -408,14 +366,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -475,14 +426,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -585,14 +529,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -851,11 +788,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -900,11 +833,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1020,14 +949,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1082,14 +1004,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1245,11 +1160,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1294,11 +1205,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1476,14 +1383,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1556,14 +1456,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1651,7 +1544,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1788,7 +1681,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1892,7 +1785,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1931,14 +1824,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1984,7 +1870,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2023,14 +1909,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2107,14 +1986,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2225,11 +2097,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2383,7 +2251,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2499,7 +2367,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2540,11 +2408,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2584,7 +2448,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2626,14 +2490,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2709,14 +2566,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework_aws/m4/manager_appframework_test.go index 03e41df85..789f9a887 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework_aws/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,27 +60,9 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { @@ -191,14 +163,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -272,14 +237,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -407,14 +365,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -474,14 +425,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -584,14 +528,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -850,11 +787,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -899,11 +832,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1019,14 +948,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1081,14 +1003,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1243,11 +1158,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1292,11 +1203,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1474,14 +1381,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1554,14 +1454,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1649,7 +1542,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager @@ -1786,7 +1679,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager @@ -1890,7 +1783,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1929,14 +1822,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1982,7 +1868,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2021,14 +1907,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2105,14 +1984,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2223,11 +2095,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2381,7 +2249,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2497,7 +2365,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2582,7 +2450,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2624,14 +2492,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2707,14 +2568,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/m4/test_helpers_test.go b/test/appframework_aws/m4/test_helpers_test.go new file mode 100644 index 000000000..40cf1603a --- /dev/null +++ b/test/appframework_aws/m4/test_helpers_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package m4appfw + +import ( + "context" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifyM4ClusterReady verifies indexers are ready, cluster is configured as multisite, and SHC is ready. +func verifyM4ClusterReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, siteCount int) { + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) +} + +// verifyM4IndexersAndSHCReady verifies indexers are ready and SHC is ready (without multisite check). +func verifyM4IndexersAndSHCReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, siteCount int) { + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) +} diff --git a/test/appframework_aws/s1/appframework_aws_suite_test.go b/test/appframework_aws/s1/appframework_aws_suite_test.go index 252889490..32bf5f13b 100644 --- a/test/appframework_aws/s1/appframework_aws_suite_test.go +++ b/test/appframework_aws/s1/appframework_aws_suite_test.go @@ -25,15 +25,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +32,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,47 +40,18 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Running "+testSuiteName) + sc, _ := GinkgoConfiguration() + sc.Timeout = 240 * time.Minute + + RunSpecs(t, "Running "+testSuiteName, sc) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/s1/appframework_aws_test.go b/test/appframework_aws/s1/appframework_aws_test.go index bf91e80ce..c41e1658b 100644 --- a/test/appframework_aws/s1/appframework_aws_test.go +++ b/test/appframework_aws/s1/appframework_aws_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -46,43 +45,16 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -773,7 +745,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to S3 @@ -829,7 +801,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from S3 testcaseEnvInst.Log.Info("Download updated ES app from S3") - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to S3 for Standalone @@ -884,7 +856,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1148,7 +1120,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download the extra apps from S3 for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 for first Standalone @@ -1278,7 +1250,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1357,7 +1329,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1447,7 +1419,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1752,7 +1724,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to S3 for Standalone @@ -1844,7 +1816,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 @@ -1911,7 +1883,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to S3 for Standalone diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go index 313c7c4fc..83875f709 100644 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ b/test/appframework_az/c3/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurec3appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go index c4c6b4eff..98459a8c4 100644 --- a/test/appframework_az/c3/appframework_azure_test.go +++ b/test/appframework_az/c3/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -49,43 +48,14 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { azureBlobClient := &testenv.AzureBlobClient{} azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -193,20 +163,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -267,19 +229,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -407,20 +362,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -477,19 +424,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -587,11 +527,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -674,12 +610,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas), "main", 2000) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -850,11 +781,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -900,11 +827,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -950,7 +873,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -1037,11 +960,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1071,7 +990,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1087,11 +1006,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1221,11 +1136,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1302,11 +1213,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1448,11 +1355,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1517,11 +1420,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1577,7 +1476,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1616,11 +1515,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1743,11 +1638,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1806,11 +1697,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1956,11 +1843,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2006,11 +1889,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2182,11 +2061,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2259,11 +2134,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2355,7 +2226,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2485,7 +2356,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2589,7 +2460,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2632,11 +2503,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2684,7 +2551,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2727,11 +2594,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2813,11 +2676,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2901,7 +2760,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2966,11 +2825,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3056,11 +2911,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3104,7 +2955,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3150,11 +3001,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3234,11 +3081,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go index 8a8a1ec5c..533bc20dc 100644 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ b/test/appframework_az/c3/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -48,43 +47,14 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { azureBlobClient := &testenv.AzureBlobClient{} azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -191,20 +161,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -265,19 +227,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -405,20 +360,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -475,19 +422,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -585,11 +525,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -672,12 +608,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas), "main", 2000) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -848,11 +779,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -898,11 +825,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -948,7 +871,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -1035,11 +958,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1069,7 +988,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1085,11 +1004,7 @@ var _ = Describe("c3appfw test", func() { // // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1220,11 +1135,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1301,11 +1212,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1447,11 +1354,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1517,11 +1420,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1577,7 +1476,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1616,11 +1515,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1743,11 +1638,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1806,11 +1697,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1956,11 +1843,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2006,11 +1889,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2182,11 +2061,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2259,11 +2134,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2355,7 +2226,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2485,7 +2356,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2589,7 +2460,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2632,11 +2503,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2684,7 +2551,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2727,11 +2594,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2813,11 +2676,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2901,7 +2760,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2966,11 +2825,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3056,11 +2911,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3104,7 +2955,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3150,11 +3001,7 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3234,11 +3081,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/c3/test_helpers_test.go b/test/appframework_az/c3/test_helpers_test.go new file mode 100644 index 000000000..f1d418225 --- /dev/null +++ b/test/appframework_az/c3/test_helpers_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package azurec3appfw + +import ( + "context" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifyC3ClusterReady verifies SHC is ready and single-site indexers are ready. +func verifyC3ClusterReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment) { + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// verifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func verifyMCVersionChangedAndReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) { + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go index 6184e543b..a416bedc9 100644 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ b/test/appframework_az/m4/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurem4appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -53,53 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go index ce5cba659..1a5afe208 100644 --- a/test/appframework_az/m4/appframework_azure_test.go +++ b/test/appframework_az/m4/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,28 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { azureBlobClient := &testenv.AzureBlobClient{} azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { @@ -192,23 +164,12 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -262,14 +223,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -397,14 +351,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -465,14 +412,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -575,14 +515,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -823,11 +756,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -873,11 +802,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -993,14 +918,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1055,14 +973,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1217,11 +1128,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1267,11 +1174,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1449,14 +1352,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1530,14 +1426,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1625,7 +1514,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1762,7 +1651,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1867,7 +1756,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1907,14 +1796,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1960,7 +1842,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2000,14 +1882,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2085,14 +1960,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2205,11 +2073,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2364,7 +2228,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2426,14 +2290,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2518,11 +2375,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2415,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2605,14 +2458,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2688,14 +2534,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go index 96d24efc2..405fb820e 100644 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ b/test/appframework_az/m4/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,16 +50,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -69,28 +59,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { azureBlobClient := &testenv.AzureBlobClient{} azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { @@ -191,23 +163,12 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -261,14 +222,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -396,14 +350,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -464,14 +411,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -574,14 +514,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -822,11 +755,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -872,11 +801,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -992,14 +917,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1054,14 +972,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1216,11 +1127,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1266,11 +1173,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1448,14 +1351,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1529,14 +1425,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1624,7 +1513,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1762,7 +1651,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1867,7 +1756,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1907,14 +1796,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1960,7 +1842,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2000,14 +1882,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2085,14 +1960,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2205,11 +2073,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2364,7 +2228,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2426,14 +2290,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2518,11 +2375,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2415,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2605,14 +2458,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2688,14 +2534,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/m4/test_helpers_test.go b/test/appframework_az/m4/test_helpers_test.go new file mode 100644 index 000000000..9e23a067c --- /dev/null +++ b/test/appframework_az/m4/test_helpers_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package azurem4appfw + +import ( + "context" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifyM4ClusterReady verifies indexers are ready, cluster is configured as multisite, and SHC is ready. +func verifyM4ClusterReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, siteCount int) { + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) +} + +// verifyM4IndexersAndSHCReady verifies indexers are ready and SHC is ready (without multisite check). +func verifyM4IndexersAndSHCReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, siteCount int) { + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) +} + +// verifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func verifyMCVersionChangedAndReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) { + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go index d3c06c3c4..3eff208c0 100644 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ b/test/appframework_az/s1/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azures1appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go index 92736ef92..cb084e554 100644 --- a/test/appframework_az/s1/appframework_azure_test.go +++ b/test/appframework_az/s1/appframework_azure_test.go @@ -20,7 +20,6 @@ import ( "strings" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -42,45 +41,17 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") azTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { azureBlobClient := &testenv.AzureBlobClient{} azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -751,7 +722,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -805,7 +776,7 @@ var _ = Describe("s1appfw test", func() { azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -861,7 +832,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1115,7 +1086,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1246,7 +1217,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1326,7 +1297,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1397,7 +1368,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1707,7 +1678,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1866,7 +1837,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big app") diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go index 9aa061bad..c73826a7a 100644 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -52,51 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go index 3a189f7a0..e49b34bd9 100644 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_test.go @@ -17,7 +17,6 @@ import ( "context" //"encoding/json" "fmt" - "path/filepath" //"strings" //"time" @@ -26,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -54,44 +52,14 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") testenv.SpecifiedTestTimeout = 5000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -197,20 +165,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -278,19 +238,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -418,20 +371,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -487,19 +432,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + verifyMCVersionChangedAndReady(ctx, testcaseEnvInst, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -585,11 +523,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -634,11 +568,7 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -716,11 +646,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + verifyC3ClusterReady(ctx, testcaseEnvInst, deployment) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go index 7c10a6b87..7d2ceabb7 100644 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ b/test/appframework_gcp/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,45 +50,14 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") testenv.SpecifiedTestTimeout = 100000 }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -839,12 +807,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas), "main", 2000) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -1559,7 +1522,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Gcs testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -2333,7 +2296,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Gcs appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to Gcs for Cluster Manager @@ -2462,7 +2425,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Gcs appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to Gcs for Cluster Manager @@ -2565,7 +2528,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Gcs appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to Gcs for Indexer Cluster @@ -2659,7 +2622,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Gcs appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to Gcs for Indexer Cluster @@ -2872,7 +2835,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to Gcs for Indexer Cluster @@ -2989,7 +2952,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to Gcs for Indexer Cluster @@ -3078,7 +3041,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Gcs appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to Gcs for Indexer Cluster @@ -3302,14 +3265,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Gcs") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") // Download Technology add-on app from Gcs testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") // Create directory for file upload to Gcs diff --git a/test/appframework_gcp/c3/test_helpers_test.go b/test/appframework_gcp/c3/test_helpers_test.go new file mode 100644 index 000000000..ea19a251d --- /dev/null +++ b/test/appframework_gcp/c3/test_helpers_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package c3gcpappfw + +import ( + "context" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifyC3ClusterReady verifies SHC is ready and single-site indexers are ready. +func verifyC3ClusterReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment) { + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// verifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func verifyMCVersionChangedAndReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) { + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go index 8f4a28249..b2c039d50 100644 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,52 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go index 96759b180..26ccdaed3 100644 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -53,16 +52,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,27 +61,9 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { @@ -192,14 +164,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -273,14 +238,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -408,14 +366,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -475,14 +426,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -585,14 +529,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -851,11 +788,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -900,11 +833,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1020,14 +949,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1082,14 +1004,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1245,11 +1160,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1294,11 +1205,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1476,14 +1383,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1556,14 +1456,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1651,7 +1544,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1788,7 +1681,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1892,7 +1785,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1931,14 +1824,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1984,7 +1870,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -2023,14 +1909,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2107,14 +1986,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2225,11 +2097,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2383,7 +2251,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2499,7 +2367,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2540,11 +2408,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4IndexersAndSHCReady(ctx, testcaseEnvInst, deployment, siteCount) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2584,7 +2448,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2626,14 +2490,7 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2709,14 +2566,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + verifyM4ClusterReady(ctx, testcaseEnvInst, deployment, siteCount) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go index bc6837431..d0abfec1e 100644 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ b/test/appframework_gcp/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,27 +60,9 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { @@ -1650,7 +1622,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1786,7 +1758,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1890,7 +1862,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1982,7 +1954,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -2381,7 +2353,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2497,7 +2469,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2582,7 +2554,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster diff --git a/test/appframework_gcp/m4/test_helpers_test.go b/test/appframework_gcp/m4/test_helpers_test.go new file mode 100644 index 000000000..e2e110af9 --- /dev/null +++ b/test/appframework_gcp/m4/test_helpers_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package m4gcpappfw + +import ( + "context" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifyM4ClusterReady verifies indexers are ready, cluster is configured as multisite, and SHC is ready. +func verifyM4ClusterReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, siteCount int) { + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) +} + +// verifyM4IndexersAndSHCReady verifies indexers are ready and SHC is ready (without multisite check). +func verifyM4IndexersAndSHCReady(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, siteCount int) { + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) +} diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go index af2fab4c2..90a237aa0 100644 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,47 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go index edfd8019f..5704e1560 100644 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -46,43 +45,16 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, func() { testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + }, filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -773,7 +745,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from gcs") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to gcs @@ -829,7 +801,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from gcs testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to gcs for Standalone @@ -884,7 +856,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from gcs testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1136,7 +1108,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from GCS testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to GCS for first Standalone @@ -1266,7 +1238,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1345,7 +1317,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1435,7 +1407,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1740,7 +1712,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to GCS for Standalone @@ -1832,7 +1804,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCS @@ -1899,7 +1871,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to GCS for Standalone diff --git a/test/custom_resource_crud/crud_test_shared.go b/test/custom_resource_crud/crud_test_shared.go new file mode 100644 index 000000000..0309d0117 --- /dev/null +++ b/test/custom_resource_crud/crud_test_shared.go @@ -0,0 +1,286 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunS1CPUUpdateTest runs the standard S1 CPU limit update test workflow +func RunS1CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string, newCPULimits string) { + // Deploy and verify Standalone + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), deployment.GetName(), "") + + // Verify telemetry + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Verify CPU limits before updating the CR + standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) + + // Change CPU limits to trigger CR update + standalone.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + err := deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") + + // Verify Standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Verify Standalone goes to ready state + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify CPU limits after updating the CR + testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) +} + +// RunC3CPUUpdateTest runs the standard C3 CPU limit update test workflow +func RunC3CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Verify cluster is ready, RF/SF is met, and MC is ready + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.StandardC3Verification(ctx, deployment, deployment.GetName(), testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "")) + + // Verify CPU limits on Indexers before updating the CR + indexerCount := 3 + testcaseEnvInst.VerifyIndexerCPULimits(deployment, deployment.GetName(), indexerCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) + testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to get instance of indexer cluster") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR") + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-idxc" + testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) + + // Verify Indexers go to ready state + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Verify CPU limits on Indexers after updating the CR + testcaseEnvInst.VerifyIndexerCPULimits(deployment, deployment.GetName(), indexerCount, newCPULimits) + + // Verify CPU limits on Search Heads before updating the CR + searchHeadCount := 3 + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + shc := &enterpriseApi.SearchHeadCluster{} + instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) + testenv.GetInstanceWithExpect(ctx, deployment, shc, instanceName, "Unable to fetch Search Head Cluster deployment") + + shc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR") + + // Verify Search Head Cluster is updating + testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + + // Verify Search Head go to ready state + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify CPU limits on Search Heads after updating the CR + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, newCPULimits) +} + +// RunC3PVCDeletionTest runs the standard C3 PVC deletion test workflow +func RunC3PVCDeletionTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, verificationTimeout time.Duration) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Verify cluster is ready and RF/SF is met + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyClusterReadyAndRFSF(ctx, deployment) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + + clusterManagerType := config.ClusterManagerPVCType() + verifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, true, verificationTimeout) + + // Delete the Search Head Cluster + shc := &enterpriseApi.SearchHeadCluster{} + err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) + Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) + err = deployment.DeleteCR(ctx, shc) + Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) + + // Delete the Indexer Cluster + idxc := &enterpriseApi.IndexerCluster{} + err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) + Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) + err = deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) + + // Delete the Cluster Manager (v3 or v4) + config.DeleteClusterManager(ctx, deployment) + + // Delete Monitoring Console + err = deployment.GetInstance(ctx, mcRef, mc) + Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) + err = deployment.DeleteCR(ctx, mc) + Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) + + verifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, false, verificationTimeout) + + // Verify Monitoring Console PVCs (etc and var) have been deleted + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) +} + +func verifyC3ClusterPVCs(testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, clusterManagerType string, exists bool, timeout time.Duration) { + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, clusterManagerType, 1, exists, timeout) +} + +// RunSHCDeployerResourceSpecTest deploys a Search Head Cluster, verifies default CPU limits, +// updates the deployer resource spec, and verifies the deployer is reconfigured while search heads retain defaults. +func RunSHCDeployerResourceSpecTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string) { + shcName := fmt.Sprintf("%s-shc", deployment.GetName()) + _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") + if err != nil { + Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "shc", shcName) + } + + // Verify CPU limits on Search Heads and deployer before updating CR + searchHeadCount := 3 + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + DeployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) + testcaseEnvInst.VerifyCPULimits(deployment, DeployerPodName, defaultCPULimits) + + shc := &enterpriseApi.SearchHeadCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, shc, shcName, "Unable to fetch Search Head Cluster deployment") + + // Assign new resources for deployer pod only + newCPULimits := "4" + newCPURequests := "2" + newMemoryLimits := "14Gi" + newMemoryRequests := "12Gi" + + depResSpec := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse(newCPURequests), + "memory": resource.MustParse(newMemoryRequests), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + "memory": resource.MustParse(newMemoryLimits), + }, + } + shc.Spec.DeployerResourceSpec = depResSpec + + testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR") + + // Verify Search Head go to ready state + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify CPU limits on Search Heads - Should be same as before + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + // Verify modified deployer spec + testcaseEnvInst.VerifyResourceConstraints(deployment, DeployerPodName, depResSpec) +} + +// RunM4CPUUpdateTest runs the standard M4 CPU limit update test workflow +func RunM4CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Multisite Cluster and Search Head Clusters + mcRef := deployment.GetName() + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + siteCount := 3 + var err error + + err = config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Ensure that the cluster-manager goes to Ready phase + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy Monitoring Console CRD + mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify RF SF is met + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Verify CPU limits on Indexers before updating the CR + for i := 1; i <= siteCount; i++ { + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) + testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) + } + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + for i := 1; i <= siteCount; i++ { + siteName := fmt.Sprintf("site%d", i) + instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) + testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to fetch Indexer Cluster deployment") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR") + } + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-" + "site1" + testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) + + // Verify Indexers go to ready state + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify RF SF is met + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Verify CPU limits after updating the CR + testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, deployment.GetName(), siteCount, newCPULimits) +} diff --git a/test/custom_resource_crud/custom_resource_crud_c3_test.go b/test/custom_resource_crud/custom_resource_crud_c3_test.go deleted file mode 100644 index 8c47e63b2..000000000 --- a/test/custom_resource_crud/custom_resource_crud_c3_test.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Master - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_m4_test.go b/test/custom_resource_crud/custom_resource_crud_m4_test.go deleted file mode 100644 index 4c00a07dc..000000000 --- a/test/custom_resource_crud/custom_resource_crud_m4_test.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_s1_test.go b/test/custom_resource_crud/custom_resource_crud_s1_test.go deleted file mode 100644 index deb487628..000000000 --- a/test/custom_resource_crud/custom_resource_crud_s1_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1)", func() { - It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { - - // Deploy Standalone - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits before updating the CR - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) - - // Change CPU limits to trigger CR update - standalone.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") - - // Verify Standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits after updating the CR - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_suite_test.go b/test/custom_resource_crud/custom_resource_crud_suite_test.go index 8972ac76e..e97331116 100644 --- a/test/custom_resource_crud/custom_resource_crud_suite_test.go +++ b/test/custom_resource_crud/custom_resource_crud_suite_test.go @@ -24,12 +24,12 @@ import ( ) const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond + // DefaultCPULimits is the default CPU limit + DefaultCPULimits = "4" + // UpdatedCPULimits is the updated CPU limit + UpdatedCPULimits = "2" + // DefaultVerificationTimeout is the default timeout for CRUD verification steps + DefaultVerificationTimeout = 150 * time.Second ) var ( @@ -39,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/custom_resource_crud/custom_resource_crud_test.go b/test/custom_resource_crud/custom_resource_crud_test.go new file mode 100644 index 000000000..766a89b1b --- /dev/null +++ b/test/custom_resource_crud/custom_resource_crud_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Crcrud test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var defaultCPULimits string + var newCPULimits string + var verificationTimeout time.Duration + + ctx := context.TODO() + + // S1 test — single variant (manager, V4) + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { + RunS1CPUUpdateTest(ctx, deployment, testcaseEnvInst, defaultCPULimits, newCPULimits) + }) + }) + + // C3 tests — V3 (master) and V4 (manager) variants + c3CrudConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastercrcrud", testenv.NewClusterReadinessConfigV3}, + {"", "managercrcrud", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range c3CrudConfigs { + tc := tc + Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + verificationTimeout = DefaultVerificationTimeout + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { + config := tc.newConfig() + RunC3CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + + It(tc.label+", integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { + config := tc.newConfig() + RunC3PVCDeletionTest(ctx, deployment, testcaseEnvInst, config, verificationTimeout) + }) + }) + } + + // CSPL-3256 - SHC deployer resource spec test (IDXC is irrelevant for this test case) + Context("Search Head Cluster", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { + RunSHCDeployerResourceSpecTest(ctx, deployment, testcaseEnvInst, defaultCPULimits) + }) + }) + + // M4 tests — V3 (master) and V4 (manager) variants + m4CrudConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastercrcrud", testenv.NewClusterReadinessConfigV3}, + {"", "managercrcrud", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range m4CrudConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration, m4: can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { + config := tc.newConfig() + RunM4CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + }) + } +}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go b/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go deleted file mode 100644 index 86bcea539..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Search Head Cluster", func() { - // CSPL-3256 - Adding the SHC only test case under c3 as IDXC is irrelevant for this test case - It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") - if err != nil { - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "Shc", shcName) - } - - // Verify CPU limits on Search Heads and deployer before updating CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - DeployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) - testcaseEnvInst.VerifyCPULimits(deployment, DeployerPodName, defaultCPULimits) - - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - // Assign new resources for deployer pod only - newCPULimits = "4" - newCPURequests := "2" - newMemoryLimits := "14Gi" - newMemoryRequests := "12Gi" - - depResSpec := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": resource.MustParse(newCPURequests), - "memory": resource.MustParse(newMemoryRequests), - }, - Limits: corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - "memory": resource.MustParse(newMemoryLimits), - }, - } - shc.Spec.DeployerResourceSpec = depResSpec - - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify CPU limits on Search Heads - Should be same as before - searchHeadCount = 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Verify modified deployer spec - testcaseEnvInst.VerifyResourceConstraints(deployment, DeployerPodName, depResSpec) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Manager - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go b/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go deleted file mode 100644 index 89fadcbb8..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/delete_cr/deletecr_suite_test.go b/test/delete_cr/deletecr_suite_test.go index 49b34af39..100ac34fc 100644 --- a/test/delete_cr/deletecr_suite_test.go +++ b/test/delete_cr/deletecr_suite_test.go @@ -1,23 +1,20 @@ // Copyright (c) 2018-2022 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "deletecr-" + testenv.RandomDNSName(3) @@ -41,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/delete_cr/deletecr_test.go b/test/delete_cr/deletecr_test.go index 2f3667ba1..92e65c0ce 100644 --- a/test/delete_cr/deletecr_test.go +++ b/test/delete_cr/deletecr_test.go @@ -1,31 +1,24 @@ // Copyright (c) 2018-2022 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" + "github.com/splunk/splunk-operator/test/testenv" ) var _ = Describe("DeleteCR test", func() { @@ -35,104 +28,22 @@ var _ = Describe("DeleteCR test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1 - Standalone Pod)", func() { It("integration, managerdeletecr: can deploy standalone and delete", func() { - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Delete Standalone CR - err = deployment.DeleteCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to Delete Standalone") - + testenv.RunDeleteStandaloneWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName()) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3)", func() { It("integration, managerdeletecr: can deploy C3 and delete search head, clustermanager", func() { - - // Deploy C3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, true, "") - Expect(err).To(Succeed(), "Unable to deploy C3 instance") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - idxc := &enterpriseApi.IndexerCluster{} - idxcName := deployment.GetName() + "-idxc" - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Unable to get Indexer instance") - - // Delete Indexer Cluster CR - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to Delete Indexer Cluster") - - sh := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, sh) - Expect(err).To(Succeed(), "Unable to get Search Head instance") - - // Delete Search Head Cluster CR - err = deployment.DeleteCR(ctx, sh) - Expect(err).To(Succeed(), "Unable to Delete Search Head Cluster") - - cm := &enterpriseApi.ClusterManager{} - cmName := deployment.GetName() - err = deployment.GetInstance(ctx, cmName, cm) - Expect(err).To(Succeed(), "Unable to get Cluster Manager instance") - - // Delete Cluster Manager CR - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to Delete Cluster Manager") - + testenv.RunDeleteC3Workflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 3) }) }) }) diff --git a/test/example/example1_test.go b/test/example/example1_test.go index bba0023d1..1ea0ffc95 100644 --- a/test/example/example1_test.go +++ b/test/example/example1_test.go @@ -14,12 +14,10 @@ package example import ( - "fmt" "math/rand" "time" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,20 +29,11 @@ var _ = XDescribe("Example1", func() { // This is invoke for each "It" spec below BeforeEach(func() { - var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) // "It" spec diff --git a/test/example/example2_test.go b/test/example/example2_test.go index 3988e0976..b692860f6 100644 --- a/test/example/example2_test.go +++ b/test/example/example2_test.go @@ -14,12 +14,10 @@ package example import ( - "fmt" "math/rand" "time" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,20 +29,11 @@ var _ = XDescribe("Example2", func() { // This is invoke for each "It" spec below BeforeEach(func() { - var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) // "It" spec diff --git a/test/example/example_suite_test.go b/test/example/example_suite_test.go index 78f198f31..183ff9af1 100644 --- a/test/example/example_suite_test.go +++ b/test/example/example_suite_test.go @@ -14,9 +14,7 @@ package example import ( - "math/rand" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -29,12 +27,7 @@ var ( testSuiteName = "example-" + testenv.RandomDNSName(3) ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - func TestExampleSuite(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -47,5 +40,7 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + if testenvInstance != nil { + Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + } }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go index d2c4be9f1..3a57cf228 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go @@ -11,13 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,18 +25,9 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv - testSuiteName = "indingsep-" + testenv.RandomDNSName(3) + testSuiteName = "indexingestionsep-" + testenv.RandomDNSName(3) queue = enterpriseApi.QueueSpec{ Provider: "sqs", @@ -100,7 +90,6 @@ var ( appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) s3TestDir = "icappfw-" + testenv.RandomDNSName(4) appListV1 = testenv.BasicApps - s3AppDirV1 = testenv.AppLocationV1 ) // TestBasic is the main entry point @@ -119,7 +108,7 @@ var _ = BeforeSuite(func() { appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download V1 app files") }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go index 9142c9882..2b1b1b4e3 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "context" @@ -21,38 +21,23 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/onsi/ginkgo/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("indingsep test", func() { +var _ = Describe("Index and Ingestion Separation test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var cmSpec enterpriseApi.ClusterManagerSpec - ctx := context.TODO() BeforeEach(func() { - var err error - - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") cmSpec = enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -65,101 +50,24 @@ var _ = Describe("indingsep test", func() { }) AfterEach(func() { - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to get Indexer Cluster instance", "Indexer Cluster Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) + setupIngestorStack(ctx, deployment, testcaseEnvInst, queue, objectStorage, cmSpec) - // Delete the Ingestor Cluster - ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Unable to get Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - err = deployment.DeleteCR(ctx, ingest) - Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - - // Delete the Queue - q = &enterpriseApi.Queue{} - err = deployment.GetInstance(ctx, "queue", q) - Expect(err).To(Succeed(), "Unable to get Queue instance", "Queue Name", q) - err = deployment.DeleteCR(ctx, q) - Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) - - // Delete the ObjectStorage - objStorage = &enterpriseApi.ObjectStorage{} - err = deployment.GetInstance(ctx, "os", objStorage) - Expect(err).To(Succeed(), "Unable to get ObjectStorage instance", "ObjectStorage Name", objStorage) - err = deployment.DeleteCR(ctx, objStorage) - Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) + deleteIngestorStack(ctx, deployment) }) }) Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") @@ -260,72 +168,23 @@ var _ = Describe("indingsep test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // Verify probe configuration - testcaseEnvInst.Log.Info("Get config map for probes") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for probes", "ConfigMap", ConfigMapName) - testcaseEnvInst.Log.Info("Verify probe configurations on Ingestor pods") - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, true) }) }) Context("Ingestor and Indexer deployment", func() { - It("indingsep, integration, indingsep: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { + It("indexingestionsep, integration: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + setupIngestorStack(ctx, deployment, testcaseEnvInst, queue, objectStorage, cmSpec) // Get instance of current Ingestor Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Ingestor Cluster CR with latest config") ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) + err := deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) Expect(err).To(Succeed(), "Failed to get instance of Ingestor Cluster") // Verify Ingestor Cluster Status @@ -388,3 +247,63 @@ var _ = Describe("indingsep test", func() { }) }) }) + +// setupIngestorStack deploys the full Queue/ObjectStorage/IngestorCluster/ClusterManager/IndexerCluster stack +// and verifies each component reaches the Ready phase. +func setupIngestorStack(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec, cmSpec enterpriseApi.ClusterManagerSpec) { + volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} + qSpec.SQS.VolList = volumeSpec + + q, err := deployment.DeployQueue(ctx, "queue", qSpec) + Expect(err).To(Succeed(), "Unable to deploy Queue") + + objStorage, err := deployment.DeployObjectStorage(ctx, "os", osSpec) + Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") + + _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") + Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") + + _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") + Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") + + testcaseEnvInst.VerifyIngestorReady(ctx, deployment) + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// deleteIngestorStack tears down the full Queue/ObjectStorage/IngestorCluster/IndexerCluster stack. +func deleteIngestorStack(ctx context.Context, deployment *testenv.Deployment) { + // Delete the Indexer Cluster + idxc := &enterpriseApi.IndexerCluster{} + err := deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) + Expect(err).To(Succeed(), "Unable to get Indexer Cluster instance", "Indexer Cluster Name", idxc) + err = deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) + + // Delete the Ingestor Cluster + ingest := &enterpriseApi.IngestorCluster{} + err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) + Expect(err).To(Succeed(), "Unable to get Ingestor Cluster instance", "Ingestor Cluster Name", ingest) + err = deployment.DeleteCR(ctx, ingest) + Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) + + // Delete the Queue + q := &enterpriseApi.Queue{} + err = deployment.GetInstance(ctx, "queue", q) + Expect(err).To(Succeed(), "Unable to get Queue instance", "Queue Name", q) + err = deployment.DeleteCR(ctx, q) + Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) + + // Delete the ObjectStorage + objStorage := &enterpriseApi.ObjectStorage{} + err = deployment.GetInstance(ctx, "os", objStorage) + Expect(err).To(Succeed(), "Unable to get ObjectStorage instance", "ObjectStorage Name", objStorage) + err = deployment.DeleteCR(ctx, objStorage) + Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) +} diff --git a/test/ingest_search/ingest_search_suite_test.go b/test/ingest_search/ingest_search_suite_test.go index 301f11611..eb4132142 100644 --- a/test/ingest_search/ingest_search_suite_test.go +++ b/test/ingest_search/ingest_search_suite_test.go @@ -15,7 +15,6 @@ package ingestsearchtest import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "ingest-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/ingest_search/ingest_search_test.go b/test/ingest_search/ingest_search_test.go index 090f034a9..2d5a21978 100644 --- a/test/ingest_search/ingest_search_test.go +++ b/test/ingest_search/ingest_search_test.go @@ -14,20 +14,9 @@ package ingestsearchtest import ( - "bufio" "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -36,231 +25,23 @@ var _ = Describe("Ingest and Search Test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var firstLine string ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can search internal logs for standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - searchString := "index=_internal | stats count by host" - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) - - var searchResults map[string]interface{} - unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - if unmarshalErr != nil { - testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") - } - - prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - searchString := "index=_internal GUID component=ServerConfig" - - // Perform a simple search - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) - if reqErr != nil { - testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - if statusErr != nil { - testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - if resErr != nil { - testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - - // Display results for debug purposes - prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + RunS1InternalLogSearchTest(ctx, deployment, testcaseEnvInst) }) - }) - Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can ingest custom data to new index and search", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify splunk status is up - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - splunkBin := "/opt/splunk/bin/splunk" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - splunkCmd := "status" - - statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) - command := []string{"/bin/bash"} - statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) - return enterpriseApi.PhaseError - } - - if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { - testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) - return enterpriseApi.PhaseError - } - - testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - // Create an index - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - indexName := "myTestIndex" - - // Create an index on a standalone instance - err = testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed response to add index to splunk") - - // Create a mock logfile to ingest - logFile := "/tmp/test.log" - err = testenv.CreateMockLogfile(logFile, 1) - Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) - - // Copy log file and ingest it - err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) - - // Read first line to find a search token - var file, openErr = os.Open(logFile) - Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) - - reader := bufio.NewReader(file) - var readErr error - firstLine, readErr = reader.ReadString('\n') - Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) - - tokens := strings.Fields(firstLine) - Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) - - searchToken := tokens[len(tokens)-1] - testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) - - searchString := fmt.Sprintf("index=%s | stats count by host", indexName) - - // Wait for search results to be available instead of fixed sleep - err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for search results") - - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) - - // Verify result. Should get count 1. result:{count:1} - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) - - hostCount := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) - testHostCnt := strings.Compare(hostCount["count"].(string), "1") - testHostname := strings.Compare(hostCount["host"].(string), podName) - Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) - - searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) - Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) - - testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) - var searchResults2 testenv.SearchJobResultsResponse - jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - found := false - for key, elem := range searchResults2.Results { - testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) - trimFirstLine := strings.TrimSuffix(firstLine, "\n") - if strings.Compare(elem.Raw, trimFirstLine) == 0 { - testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) - found = true - } - } - Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) + RunS1IngestAndSearchTest(ctx, deployment, testcaseEnvInst) }) }) }) diff --git a/test/ingest_search/ingest_search_test_shared.go b/test/ingest_search/ingest_search_test_shared.go new file mode 100644 index 000000000..1dccd1b58 --- /dev/null +++ b/test/ingest_search/ingest_search_test_shared.go @@ -0,0 +1,210 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ingestsearchtest + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// RunS1InternalLogSearchTest deploys a Standalone instance and verifies internal log searches +// using both synchronous and asynchronous search APIs. +func RunS1InternalLogSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + + searchString := "index=_internal | stats count by host" + searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) + + var searchResults map[string]interface{} + unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + if unmarshalErr != nil { + testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") + } + + prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + searchString := "index=_internal GUID component=ServerConfig" + + sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) + if reqErr != nil { + testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) + if statusErr != nil { + testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) + if resErr != nil { + testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + + prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) +} + +// RunS1IngestAndSearchTest deploys a Standalone instance, ingests a custom log file into a new +// index, and verifies the ingested data is searchable via both sync and async search APIs. +func RunS1IngestAndSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + + splunkBin := "/opt/splunk/bin/splunk" + username := "admin" + password := "$(cat /mnt/splunk-secrets/password)" + splunkCmd := "status" + + statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) + command := []string{"/bin/bash"} + statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) + return enterpriseApi.PhaseError + } + + if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { + testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) + return enterpriseApi.PhaseError + } + + testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) + indexName := "myTestIndex" + + err = testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) + Expect(err).To(Succeed(), "Failed response to add index to splunk") + + logFile := "/tmp/test.log" + err = testenv.CreateMockLogfile(logFile, 1) + Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) + + err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) + Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) + + file, openErr := os.Open(logFile) + Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) + + reader := bufio.NewReader(file) + firstLine, readErr := reader.ReadString('\n') + Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) + + tokens := strings.Fields(firstLine) + Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) + + searchToken := tokens[len(tokens)-1] + testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) + + searchString := fmt.Sprintf("index=%s | stats count by host", indexName) + + err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) + Expect(err).To(Succeed(), "Timed out waiting for search results") + + searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) + + var searchResults map[string]interface{} + jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) + + hostCount := searchResults["result"].(map[string]interface{}) + testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) + testHostCnt := strings.Compare(hostCount["count"].(string), "1") + testHostname := strings.Compare(hostCount["host"].(string), podName) + Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) + + searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) + sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) + Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) + Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) + Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) + + testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) + var searchResults2 testenv.SearchJobResultsResponse + jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + found := false + for key, elem := range searchResults2.Results { + testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) + trimFirstLine := strings.TrimSuffix(firstLine, "\n") + if strings.Compare(elem.Raw, trimFirstLine) == 0 { + testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) + found = true + } + } + Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) +} diff --git a/test/licensemanager/manager_suite_test.go b/test/licensemanager/licensemanager_suite_test.go similarity index 77% rename from test/licensemanager/manager_suite_test.go rename to test/licensemanager/licensemanager_suite_test.go index d504c3685..8a58bf6da 100644 --- a/test/licensemanager/manager_suite_test.go +++ b/test/licensemanager/licensemanager_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package licensemanager import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lmanager-" + testenv.RandomDNSName(3) diff --git a/test/licensemanager/lm_appfw_test.go b/test/licensemanager/lm_appfw_test.go new file mode 100644 index 000000000..547603f9f --- /dev/null +++ b/test/licensemanager/lm_appfw_test.go @@ -0,0 +1,45 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemanager + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Licensemanager App Framework test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var config *LicenseTestConfig + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + + config = NewLicenseManagerConfig() + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { + It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { + RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) + }) + }) +}) diff --git a/test/licensemanager/lm_s1_test.go b/test/licensemanager/lm_s1_test.go deleted file mode 100644 index 035ab730f..000000000 --- a/test/licensemanager/lm_s1_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with License Manager", func() { - It("licensemanager, smoke, s1: Splunk Operator can configure License Manager with Standalone in S1 SVA", func() { - RunLMS1Test(ctx, deployment, testcaseEnvInst, config) - }) - }) -}) diff --git a/test/licensemanager/manager_lm_m4_test.go b/test/licensemanager/lm_test.go similarity index 57% rename from test/licensemanager/manager_lm_m4_test.go rename to test/licensemanager/lm_test.go index aedcd4dee..88200fb8c 100644 --- a/test/licensemanager/manager_lm_m4_test.go +++ b/test/licensemanager/lm_test.go @@ -15,12 +15,8 @@ package licensemanager import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" ) @@ -32,33 +28,25 @@ var _ = Describe("Licensemanager test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Standalone deployment (S1) with License Manager", func() { + It("licensemanager, smoke, s1: Splunk Operator can configure License Manager with Standalone in S1 SVA", func() { + RunLMS1Test(ctx, deployment, testcaseEnvInst, config) + }) + }) + + Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { + It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", func() { + RunLMC3Test(ctx, deployment, testcaseEnvInst, config) + }) }) Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Manager", func() { @@ -66,5 +54,4 @@ var _ = Describe("Licensemanager test", func() { RunLMM4Test(ctx, deployment, testcaseEnvInst, config) }) }) - }) diff --git a/test/licensemanager/lm_test_shared.go b/test/licensemanager/lm_test_shared.go index 205fd2c74..69eabf821 100644 --- a/test/licensemanager/lm_test_shared.go +++ b/test/licensemanager/lm_test_shared.go @@ -20,91 +20,77 @@ import ( "path/filepath" "time" + . "github.com/onsi/gomega" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" - - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" ) type LicenseTestConfig struct { - DeployStandaloneWithLM func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) - LicenseManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - ClusterManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - DeployMultisiteClusterWithSearchHead func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas, siteCount int, mcRef string) error - DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) - LicenseManagerPodName string - LicenseManagerSpecType string + *testenv.ClusterReadinessConfig + DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) + BuildLMAppFrameworkSpec func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} + LicenseManagerPodName string } func NewLicenseMasterConfig() *LicenseTestConfig { return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterMasterReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, + ClusterReadinessConfig: testenv.NewClusterReadinessConfigV3(), DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { return deployment.DeployLicenseMasterWithGivenSpec(ctx, name, spec.(enterpriseApiV3.LicenseMasterSpec)) }, - LicenseManagerPodName: testenv.LicenseMasterPod, - LicenseManagerSpecType: "v3", + BuildLMAppFrameworkSpec: func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApiV3.LicenseMasterSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Volumes: []corev1.Volume{{ + Name: "licenses", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: testcaseEnvInst.GetLMConfigMap(), + }, + }, + }, + }}, + LicenseURL: "/mnt/licenses/enterprise.lic", + Spec: enterpriseApi.Spec{ImagePullPolicy: "Always", Image: testcaseEnvInst.GetSplunkImage()}, + }, + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: testenv.LicenseMasterPod, } } func NewLicenseManagerConfig() *LicenseTestConfig { return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLM(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterManagerReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, + ClusterReadinessConfig: testenv.NewClusterReadinessConfigV4(), DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { return deployment.DeployLicenseManagerWithGivenSpec(ctx, name, spec.(enterpriseApi.LicenseManagerSpec)) }, - LicenseManagerPodName: testenv.LicenseManagerPod, - LicenseManagerSpecType: "v4", - } -} - -func downloadLicenseAndCreateConfigMap(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv) { - downloadDir := "licenseFolder" - - var err error - var licenseFilePath string - - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err = testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - case "azure": - licenseFilePath, err = testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - case "gcp": - licenseFilePath, err = testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + BuildLMAppFrameworkSpec: func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApi.LicenseManagerSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Volumes: []corev1.Volume{{ + Name: "licenses", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: testcaseEnvInst.GetLMConfigMap(), + }, + }, + }, + }}, + LicenseURL: "/mnt/licenses/enterprise.lic", + Spec: enterpriseApi.Spec{ImagePullPolicy: "Always", Image: testcaseEnvInst.GetSplunkImage()}, + }, + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: testenv.LicenseManagerPod, } - - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) } func downloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) { @@ -158,7 +144,7 @@ func deleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) // Create standalone Deployment with License Manager/Master mcRef := deployment.GetName() @@ -166,26 +152,16 @@ func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEn Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Deploy and verify Monitoring Console + _ = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, false) // Verify License Manager/Master is configured on standalone instance standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) @@ -198,51 +174,23 @@ func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEn func RunLMC3Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) // Deploy Single site Cluster with License Manager/Master mcRef := deployment.GetName() err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) Expect(err).To(Succeed(), "Unable to deploy cluster") - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyC3ComponentsReady(ctx, deployment) - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + deployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef) - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Verify License Manager/Master is configured on indexers and search heads + testenv.VerifyLMConfiguredOnIndexers(ctx, deployment, deployment.GetName(), 3) + testenv.VerifyLMConfiguredOnSearchHeads(ctx, deployment, deployment.GetName(), 3) - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + testenv.VerifyLMConfiguredOnMC(ctx, deployment) } func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, config *LicenseTestConfig) { @@ -275,7 +223,7 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment uploadedApps = append(uploadedApps, uploadedFiles...) // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) // Create App framework Spec volumeName := "lm-test-volume-" + testenv.RandomDNSName(3) @@ -307,54 +255,7 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment AppSources: appSourceSpec, } - var spec interface{} - if config.LicenseManagerSpecType == "v3" { - spec = enterpriseApiV3.LicenseMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } else { - spec = enterpriseApi.LicenseManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } + spec := config.BuildLMAppFrameworkSpec(testcaseEnvInst, appFrameworkSpec) // Deploy the License Manager/Master with App Framework var err error @@ -362,7 +263,7 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment Expect(err).To(Succeed(), "Unable to deploy LM with App framework") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) podName := []string{fmt.Sprintf(config.LicenseManagerPodName, deployment.GetName(), 0)} @@ -391,7 +292,7 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment Expect(err).To(Succeed(), "Timed out waiting for LicenseManager to reach Ready phase") // Verify LM stays in ready state - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV2, true, enterpriseApi.ScopeLocal) @@ -409,53 +310,27 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment func RunLMM4Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) // Deploy Multisite Cluster with License Manager/Master and Search Head siteCount := 3 mcRef := deployment.GetName() - err := config.DeployMultisiteClusterWithSearchHead(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) + err := config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) Expect(err).To(Succeed(), "Unable to deploy cluster") - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) - // Verify Multisite Indexer Cluster status - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + deployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef) - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Verify License Manager/Master is configured on indexers and search heads + testenv.VerifyLMConfiguredOnMultisiteIndexers(ctx, deployment, deployment.GetName(), siteCount) + testenv.VerifyLMConfiguredOnSearchHeads(ctx, deployment, deployment.GetName(), 3) - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testenv.VerifyLMConfiguredOnMC(ctx, deployment) +} - // Verify RF SF is met +func deployMCAndVerifyRFSF(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, mcRef string) { + _ = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 2, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 3, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) } diff --git a/test/licensemanager/manager_lm_c3_test.go b/test/licensemanager/manager_lm_c3_test.go deleted file mode 100644 index 8e44449c9..000000000 --- a/test/licensemanager/manager_lm_c3_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", func() { - RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { - RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemaster/lm_c3_test.go b/test/licensemaster/lm_appfw_test.go similarity index 50% rename from test/licensemaster/lm_c3_test.go rename to test/licensemaster/lm_appfw_test.go index ed2f83f7f..123054338 100644 --- a/test/licensemaster/lm_c3_test.go +++ b/test/licensemaster/lm_appfw_test.go @@ -15,16 +15,14 @@ package licensemaster import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" + "github.com/splunk/splunk-operator/test/licensemanager" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("licensemaster test", func() { +var _ = Describe("Licensemaster App Framework test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment @@ -32,41 +30,13 @@ var _ = Describe("licensemaster test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { - licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { diff --git a/test/licensemaster/lm_s1_test.go b/test/licensemaster/lm_s1_test.go deleted file mode 100644 index cc5c92e6b..000000000 --- a/test/licensemaster/lm_s1_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with License Master", func() { - It("licensemaster, smoke, s1: Splunk Operator can configure License Master with Standalone in S1 SVA", func() { - licensemanager.RunLMS1Test(ctx, deployment, testcaseEnvInst, config) - }) - }) -}) diff --git a/test/licensemaster/lm_suite_test.go b/test/licensemaster/lm_suite_test.go index 9df3f4576..ed5c80594 100644 --- a/test/licensemaster/lm_suite_test.go +++ b/test/licensemaster/lm_suite_test.go @@ -15,7 +15,6 @@ package licensemaster import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lm-" + testenv.RandomDNSName(3) diff --git a/test/licensemaster/lm_m4_test.go b/test/licensemaster/lm_test.go similarity index 59% rename from test/licensemaster/lm_m4_test.go rename to test/licensemaster/lm_test.go index 1e4c6fb65..c0d4911fb 100644 --- a/test/licensemaster/lm_m4_test.go +++ b/test/licensemaster/lm_test.go @@ -15,11 +15,8 @@ package licensemaster import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/licensemanager" "github.com/splunk/splunk-operator/test/testenv" @@ -33,35 +30,25 @@ var _ = Describe("Licensemaster test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) - if deployment != nil { - deployment.Teardown() - } + Context("Standalone deployment (S1) with License Master", func() { + It("licensemaster, smoke, s1: Splunk Operator can configure License Master with Standalone in S1 SVA", func() { + licensemanager.RunLMS1Test(ctx, deployment, testcaseEnvInst, config) + }) + }) - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { + It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { + licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) + }) }) Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Master", func() { @@ -69,5 +56,4 @@ var _ = Describe("Licensemaster test", func() { licensemanager.RunLMM4Test(ctx, deployment, testcaseEnvInst, config) }) }) - }) diff --git a/test/monitoring_console/manager_monitoring_console_test.go b/test/monitoring_console/manager_monitoring_console_test.go deleted file mode 100644 index 1cc3b89f3..000000000 --- a/test/monitoring_console/manager_monitoring_console_test.go +++ /dev/null @@ -1,1101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package monitoringconsoletest - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Monitoring Console test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Deploy Monitoring Console", func() { - It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { - /* - Test Steps - 1. Deploy Monitoring Console - 2. Deploy Standalone - 3. Wait for Monitoring Console status to go back to READY - 4. Verify Standalone configured in Monitoring Console Config Map - 5. Verify Monitoring Console Pod has correct peers in Peer List - --------------- RECONFIG WITH NEW MC -------------------------- - 6. Reconfig S1 with 2nd Monitoring Console Name - 7. Check 2nd Monitoring Console Config Map to verify s1 - 8. Deploy 2nd Monitoring Console Pod - 9. Verify Standalone pod is configured on Monitoring Console Pod - 10. Verify 1st Monitoring Console Config Map is not configured with S1 - 11. Verify 1st Monitoring Console Pod is not configured with S1 - */ - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Create Standalone Spec and apply - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### - - // Reconfig S1 with 2nd Monitoring Console Name - mcTwoName := deployment.GetName() + "-two" - err = deployment.GetInstance(ctx, standaloneOneName, standaloneOne) - Expect(err).To(Succeed(), "Unable to get instance of Standalone") - standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - - // Update Standalone with 2nd MC - err = deployment.UpdateCR(ctx, standaloneOne) - Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - - // Deploy 2nd MC Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Second Monitoring Console Pod") - - // Verify 2nd Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on SECOND MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcTwoName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in SECOND MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcTwoName, true, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is not configured in MC ONE Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod NOT ON FIRST MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod Not in MC ONE Peer List - testcaseEnvInst.Log.Info("Check standalone NOT ON FIRST MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - // CSPL-901 standaloneTwoName := deployment.GetName() + "-two" - standaloneTwoName := "standalone-" + testenv.RandomDNSName(3) - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete Standlone TWO of the standalone and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment with Scale up", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Scale Standalone to 2 REPLICAS - 8. Wait for Second Standalone POD to come up and PHASE to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone PODS configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 1, false, 0) - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Scale Standalone instance - testcaseEnvInst.Log.Info("Scaling Standalone CR") - scaledReplicaCount := 2 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale Standalone") - - // Ensure standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Manager to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) - - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc1, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Manager CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Manager Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo, testcaseEnvInst) - - // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") - - // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - }) - }) - - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("managermc2, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Manager to use 2nd Monitoring Console - 9. Verify Cluster Manager is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Manager not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Manager RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { - /* - Test Steps - 1. Deploy Standalone with name "search-head-adhoc" - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone with name "search-head" - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := "search-head-adhoc" - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standaloneOne to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneOneName, standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneOne is configured in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneOne Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - standaloneTwoName := "search-head" - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource(CR) resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneTwo is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneTwo Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete standaloneTwo and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - -}) diff --git a/test/monitoring_console/mc_test.go b/test/monitoring_console/mc_test.go new file mode 100644 index 000000000..4d42c4f08 --- /dev/null +++ b/test/monitoring_console/mc_test.go @@ -0,0 +1,809 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package monitoringconsoletest + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + "github.com/splunk/splunk-operator/test/testenv" +) + +// Master (V3) Monitoring Console tests +var _ = Describe("Monitoring Console test (master)", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + It("mastermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { + /* + Test Steps + 1. Deploy Single Site Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. VerifyMonitoring Console Pod has Search Heads in Peer strings + 6. Verify Monitoring Console Pod has peers(indexers) in Peer string + 7. Scale SH Cluster + 8. Scale Indexer Count + 9. Add a standalone + 10. Verify Standalone is configured in MC Config Map and Peer String + 11. Verify SH are configured in MC Config Map and Peers String + 12. Verify Indexers are configured in Peer String + */ + + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy and verify Monitoring Console + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + // Deploy and verify C3 cluster with MC + testcaseEnvInst.DeployAndVerifyC3WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, mcName) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Wait for Cluster Master to appear in Monitoring Console Config Map + err := testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for Cluster Master in MC ConfigMap") + + // Verify MC configuration for C3 cluster + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, defaultSHReplicas, defaultIndexerReplicas, true) + + // Scale Search Head Cluster + scaledSHReplicas := defaultSHReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, deployment.GetName(), scaledSHReplicas) + + // Scale indexers + scaledIndexerReplicas := defaultIndexerReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, deployment.GetName(), scaledIndexerReplicas) + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Deploy Standalone with MC reference + testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcName) + + // Ensure Indexer cluster go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Ensure Search Head Cluster go to Ready Phase + // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + + // Wait for MC to go to PENDING Phase + //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify Standalone configured on Monitoring Console + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, deployment.GetName(), mcName, true) + + // Verify MC configuration after scale up + testcaseEnvInst.Log.Info("Verify MC configuration after Scale Up") + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, scaledSHReplicas, scaledIndexerReplicas, true) + }) + }) + + Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + /* + Test Steps + 1. Deploy Single Site Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. Verify SH are configured in peer strings on MC Pod + 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod + --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- + 13. Reconfigure CM with Second MC + 14. Verify CM in config map of Second MC + 15. Create Second MC Pod + 16. Verify Indexers in second MC Pod + 17. Verify SHC not in second MC + 18. Verify SHC still present in first MC + 19. Verify CM and Indexers not in first MC Pod + ---------------- RECONFIG SHC WITH NEW MC + 20. Configure SHC with Second MC + 21. Verify CM, DEPLOYER, Search Heads in config map of second MC + 22. Verify SHC in second MC pod + 23. Verify Indexers in Second MC Pod + 24. Verify CM and Deployer not in first MC CONFIG MAP + 24. Verify SHC, Indexers not in first MC Pod + */ + + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy Monitoring Console Pod + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + err := deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + // Ensure that the cluster-master goes to Ready phase + testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.VerifyC3ComponentsReady(ctx, deployment) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Verify MC configuration for C3 cluster + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, defaultSHReplicas, defaultIndexerReplicas, true) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Generate pod name slices for later verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) + + // ################# Update Monitoring Console In Cluster Master CR ################################## + + mcTwoName := deployment.GetName() + "-two" + cm := &enterpriseApiV3.ClusterMaster{} + err = deployment.GetInstance(ctx, deployment.GetName(), cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) + + cm.Spec.MonitoringConsoleRef.Name = mcTwoName + err = deployment.UpdateCR(ctx, cm) + + Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) + + // Ensure Cluster Master Goes to Updating Phase + //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + + // Ensure that the cluster-master goes to Ready phase + testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + + // Ensure indexers go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Deploy and verify Monitoring Console Two + mcTwo := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### + masterParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterMasterServiceName, CMURLKey: "SPLUNK_CLUSTER_MASTER_URL"} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcTwoName, shPods, indexerPods, true) + + // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcName, mc, shPods, false) + + // ################# Update Monitoring Console In SHC CR ################################## + + // Update SHC to use 2nd Monitoring Console + shc := &enterpriseApi.SearchHeadCluster{} + shcName := deployment.GetName() + "-shc" + testenv.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, testcaseEnvInst, shc, shcName, mcTwoName) + + // Ensure Search Head Cluster go to Ready Phase + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify MC is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) + + // ############################ VERIFICATOIN FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + VerifyMCTwoAfterSHCReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcTwoName, shPods, indexerPods, 0) + + // ############################ VERIFICATOIN FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + VerifyMCOneAfterSHCReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcName, mc, shPods, 0) + }) + }) + + Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { + It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { + /* + Test Steps + 1. Deploy Multisite Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. Verify SH are configured in peer strings on MC Pod + 6. Verify Indexers are configured in MC Config Map + 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod + ############ CLUSTER MANAGER MC RECONFIG ################################# + 8. Configure Cluster Master to use 2nd Monitoring Console + 9. Verify Cluster Master is configured Config Maps of Second MC + 10. Deploy 2nd MC pod + 11. Verify Indexers in 2nd MC Pod + 12. Verify SHC not in 2nd MC CM + 13. Verify SHC not in 2nd MC Pod + 14. Verify Cluster Master not 1st MC Config Map + 15. Verify Indexers not in 1st MC Pod + */ + defaultSHReplicas := 3 + defaultIndexerReplicas := 1 + siteCount := 3 + mcName := deployment.GetName() + err := deployment.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) + Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with SHC") + + // Ensure that the cluster-master goes to Ready phase + testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Verify MC configuration for M4 cluster + testcaseEnvInst.VerifyMCConfigForM4Cluster(ctx, deployment, deployment.GetName(), mcName, defaultSHReplicas, defaultIndexerReplicas, siteCount, true) + + // Generate pod name slices for later verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), defaultIndexerReplicas, true, siteCount) + + // ############ CLUSTER MANAGER MC RECONFIG ################################# + mcTwoName := deployment.GetName() + "-two" + // Update Cluster Manager to use 2nd Monitoring Console + cm := &enterpriseApiV3.ClusterMaster{} + testenv.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, testcaseEnvInst, cm, deployment.GetName(), mcTwoName) + + // Ensure Cluster Master Goes to Updating Phase + //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + + // Ensure that the cluster-master goes to Ready phase + testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + + // Deploy and verify Monitoring Console Two + mcTwo := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + // Verify Monitoring Console TWO is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) + + masterParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterMasterServiceName, CMURLKey: "SPLUNK_CLUSTER_MASTER_URL"} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcTwoName, shPods, indexerPods, false) + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, masterParams, mcName, mc, shPods, true) + + }) + }) +}) + +// Manager (V4) Monitoring Console tests +var _ = Describe("Monitoring Console test (manager)", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Deploy Monitoring Console", func() { + It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { + /* + Test Steps + 1. Deploy Monitoring Console + 2. Deploy Standalone + 3. Wait for Monitoring Console status to go back to READY + 4. Verify Standalone configured in Monitoring Console Config Map + 5. Verify Monitoring Console Pod has correct peers in Peer List + --------------- RECONFIG WITH NEW MC -------------------------- + 6. Reconfig S1 with 2nd Monitoring Console Name + 7. Check 2nd Monitoring Console Config Map to verify s1 + 8. Deploy 2nd Monitoring Console Pod + 9. Verify Standalone pod is configured on Monitoring Console Pod + 10. Verify 1st Monitoring Console Config Map is not configured with S1 + 11. Verify 1st Monitoring Console Pod is not configured with S1 + */ + + // Deploy Monitoring Console CRD + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + // Create Standalone Spec and apply + standaloneOneName := deployment.GetName() + mcName := deployment.GetName() + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone to be in READY Status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Verify MC is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Check Standalone is configured in MC + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### + + // Reconfig S1 with 2nd Monitoring Console Name + mcTwoName := deployment.GetName() + "-two" + err = deployment.GetInstance(ctx, standaloneOneName, standaloneOne) + Expect(err).To(Succeed(), "Unable to get instance of Standalone") + standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName + + // Update Standalone with 2nd MC + err = deployment.UpdateCR(ctx, standaloneOne) + Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") + + // Deploy 2nd MC Pod + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + // Check Standalone is configured in MC Two + testcaseEnvInst.Log.Info("Checking for Standalone on SECOND MC after Standalone RECONFIG") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcTwoName, true) + + // Verify Monitoring Console One is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Check Standalone is NOT configured in MC One + testcaseEnvInst.Log.Info("Checking for Standalone NOT ON FIRST MC after Standalone RECONFIG") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, false) + + }) + }) + + Context("Standalone deployment (S1)", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, deployment.GetName(), "standalone-"+testenv.RandomDNSName(3)) + }) + }) + + Context("Standalone deployment with Scale up", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { + /* + Test Steps + 1. Deploy Standalone + 2. Wait for Standalone to go to READY + 3. Deploy Monitoring Console + 4. Wait for Monitoring Console status to be READY + 5. Verify Standalone configured in Monitoring Console Config Map + 6. Verify Monitoring Console Pod has correct peers in Peer List + 7. Scale Standalone to 2 REPLICAS + 8. Wait for Second Standalone POD to come up and PHASE to be READY + 9. Wait for Monitoring Console status to go UPDATING then READY + 10. Verify both Standalone PODS configured in Monitoring Console Config Map + 11. Verify both Standalone configured in Monitoring Console Pod Peers String + */ + + standaloneName := deployment.GetName() + mcName := deployment.GetName() + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone to be in READY Status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Deploy MC and wait for MC to be READY + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Check Standalone is configured in MC + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 1, false, 0) + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Scale Standalone instance + testcaseEnvInst.Log.Info("Scaling Standalone CR") + scaledReplicaCount := 2 + standalone = &enterpriseApi.Standalone{} + err = deployment.GetInstance(ctx, deployment.GetName(), standalone) + Expect(err).To(Succeed(), "Failed to get instance of Standalone") + + standalone.Spec.Replicas = int32(scaledReplicaCount) + + err = deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Failed to scale Standalone") + + // Ensure standalone is scaling up + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + + // Wait for MC to go to Updating Phase + //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Verify MC is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) + + // Check both Standalone pods are configured in MC after scale up + testcaseEnvInst.Log.Info("Checking for Standalone Pods on MC after scale up") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + }) + }) + + Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { + /* + Test Steps + 1. Deploy Single Site Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. VerifyMonitoring Console Pod has Search Heads in Peer strings + 6. Verify Monitoring Console Pod has peers(indexers) in Peer string + 7. Scale SH Cluster + 8. Scale Indexer Count + 9. Add a standalone + 10. Verify Standalone is configured in MC Config Map and Peer String + 11. Verify SH are configured in MC Config Map and Peers String + 12. Verify Indexers are configured in Peer String + */ + + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy Monitoring Console Pod + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + err := deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifyC3ComponentsReady(ctx, deployment) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Wait for Cluster Manager to appear in Monitoring Console Config Map + err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") + + // Check Deployer in Monitoring Console Config Map + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + + // Check Search Head Pods in Monitoring Console Config Map + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + + // Wait for Monitoring console Pod to be configured with all search head + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") + + // Check Monitoring console is configured with all Indexer in Name Space + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + + // Scale Search Head Cluster + scaledSHReplicas := defaultSHReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, deployment.GetName(), scaledSHReplicas) + + // Scale indexers + scaledIndexerReplicas := defaultIndexerReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, deployment.GetName(), scaledIndexerReplicas) + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Deploy Standalone Pod + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Ensure Indexer cluster go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Ensure Search Head Cluster go to Ready Phase + // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + + // Wait for MC to go to PENDING Phase + //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Check Standalone configured on Monitoring Console + standalonePods := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // Verify all Search Head Members are configured on Monitoring Console + shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) + + testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + + testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + + // Check Monitoring console is configured with all Indexer in Name Space + testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") + indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + }) + }) + + Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + It("managermc1, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + /* + Test Steps + 1. Deploy Single Site Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. Verify SH are configured in peer strings on MC Pod + 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod + --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- + 13. Reconfigure CM with Second MC + 14. Verify CM in config map of Second MC + 15. Create Second MC Pod + 16. Verify Indexers in second MC Pod + 17. Verify SHC not in second MC + 18. Verify SHC still present in first MC + 19. Verify CM and Indexers not in first MC Pod + ---------------- RECONFIG SHC WITH NEW MC + 20. Configure SHC with Second MC + 21. Verify CM, DEPLOYER, Search Heads in config map of second MC + 22. Verify SHC in second MC pod + 23. Verify Indexers in Second MC Pod + 24. Verify CM and Deployer not in first MC CONFIG MAP + 24. Verify SHC, Indexers not in first MC Pod + */ + + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy Monitoring Console Pod + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + err := deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifyC3ComponentsReady(ctx, deployment) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Check Cluster Manager in Monitoring Console Config Map + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) + + // Check Deployer in Monitoring Console Config Map + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + + // Check Search Head Pods in Monitoring Console Config Map + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Wait for Monitoring console Pod to be configured with all search head + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") + + // Check Monitoring console is configured with all Indexer in Name Space + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + + // ################# Update Monitoring Console In Cluster Manager CR ################################## + + mcTwoName := deployment.GetName() + "-two" + cm := &enterpriseApi.ClusterManager{} + err = deployment.GetInstance(ctx, deployment.GetName(), cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) + + cm.Spec.MonitoringConsoleRef.Name = mcTwoName + err = deployment.UpdateCR(ctx, cm) + + Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) + + // Ensure Cluster Manager Goes to Updating Phase + //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + + // Ensure indexers go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Deploy Monitoring Console Pod + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### + managerParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterManagerServiceName, CMURLKey: splcommon.ClusterManagerURL} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcTwoName, shPods, indexerPods, true) + + // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcName, mc, shPods, false) + + // ################# Update Monitoring Console In SHC CR ################################## + + // Get instance of current SHC CR with latest config + shc := &enterpriseApi.SearchHeadCluster{} + shcName := deployment.GetName() + "-shc" + err = deployment.GetInstance(ctx, shcName, shc) + Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + + // Update SHC to use 2nd Monitoring Console + shc.Spec.MonitoringConsoleRef.Name = mcTwoName + err = deployment.UpdateCR(ctx, shc) + Expect(err).To(Succeed(), "Failed to update Monitoring Console in Search Head Cluster CRD") + + // Ensure Search Head Cluster go to Ready Phase + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify MC is Ready and stays in ready state + // testenv.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo, testcaseEnvInst) + + // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + VerifyMCTwoAfterSHCReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcTwoName, shPods, indexerPods, 5*time.Minute) + + // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + VerifyMCOneAfterSHCReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcName, mc, shPods, 5*time.Minute) + }) + }) + + Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { + It("managermc2, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { + /* + Test Steps + 1. Deploy Multisite Indexer Cluster + 2. Deploy Monitoring Console + 3. Wait for Monitoring Console status to go back to READY + 4. Verify SH are configured in MC Config Map + 5. Verify SH are configured in peer strings on MC Pod + 6. Verify Indexers are configured in MC Config Map + 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod + ############ CLUSTER MANAGER MC RECONFIG ################################# + 8. Configure Cluster Manager to use 2nd Monitoring Console + 9. Verify Cluster Manager is configured Config Maps of Second MC + 10. Deploy 2nd MC pod + 11. Verify Indexers in 2nd MC Pod + 12. Verify SHC not in 2nd MC CM + 13. Verify SHC not in 2nd MC Pod + 14. Verify Cluster Manager not 1st MC Config Map + 15. Verify Indexers not in 1st MC Pod + */ + defaultSHReplicas := 3 + defaultIndexerReplicas := 1 + siteCount := 3 + mcName := deployment.GetName() + err := deployment.DeployMultisiteClusterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + + // Ensure indexers go to Ready phase + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + + // Ensure indexer clustered is configured as multisite + testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + + // Ensure search head cluster go to Ready phase + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Check Cluster Manager in Monitoring Console Config Map + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) + + // Check Deployer in Monitoring Console Config Map + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + + // Deploy Monitoring Console Pod + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Check Monitoring console is configured with all search head instances in namespace + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + + testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + + testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + + // Check Monitoring console is configured with all Indexer in Name Space + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) + testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + + // ############ CLUSTER MANAGER MC RECONFIG ################################# + mcTwoName := deployment.GetName() + "-two" + cm := &enterpriseApi.ClusterManager{} + err = deployment.GetInstance(ctx, deployment.GetName(), cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + + // get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) + + cm.Spec.MonitoringConsoleRef.Name = mcTwoName + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") + + // wait for custom resource resource version to change + testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) + + // Ensure Cluster Manager Goes to Updating Phase + //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + + // Ensure that the cluster-manager goes to Ready phase + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + + // Deploy Monitoring Console Pod + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + managerParams := MCReconfigParams{CMServiceNameFmt: testenv.ClusterManagerServiceName, CMURLKey: splcommon.ClusterManagerURL} + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcTwoName, shPods, indexerPods, false) + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, managerParams, mcName, mc, shPods, true) + + }) + }) + + Context("Standalone deployment (S1)", func() { + It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, "search-head-adhoc", "search-head") + }) + }) + +}) diff --git a/test/monitoring_console/monitoring_console_suite_test.go b/test/monitoring_console/monitoring_console_suite_test.go index 83bf2060d..274e507f4 100644 --- a/test/monitoring_console/monitoring_console_suite_test.go +++ b/test/monitoring_console/monitoring_console_suite_test.go @@ -1,18 +1,16 @@ // Copyright (c) 2018-2022 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package monitoringconsoletest import ( @@ -25,15 +23,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "mc-" + testenv.RandomDNSName(3) @@ -41,7 +30,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) sc, _ := GinkgoConfiguration() diff --git a/test/monitoring_console/monitoring_console_test.go b/test/monitoring_console/monitoring_console_test.go deleted file mode 100644 index 87605c825..000000000 --- a/test/monitoring_console/monitoring_console_test.go +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package monitoringconsoletest - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("Monitoring Console test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Master to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Master in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) - - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Master CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Master Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // ############################ VERIFICATOIN FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - }) - }) - - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Master to use 2nd Monitoring Console - 9. Verify Cluster Master is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Master not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with SHC") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Master RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - }) - }) -}) diff --git a/test/monitoring_console/monitoring_console_test_shared.go b/test/monitoring_console/monitoring_console_test_shared.go new file mode 100644 index 000000000..100647fd2 --- /dev/null +++ b/test/monitoring_console/monitoring_console_test_shared.go @@ -0,0 +1,245 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package monitoringconsoletest + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunS1StandaloneAddDeleteMCTest deploys two standalone instances with a Monitoring Console, +// verifies both are registered, then deletes the second standalone and verifies the MC +// config map and peer list are updated correctly. +func RunS1StandaloneAddDeleteMCTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, standaloneOneName, standaloneTwoName string) { + mcName := deployment.GetName() + + // Deploy standalone one with MCRef + spec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone to be in READY Status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneOneName, standaloneOne) + + // Deploy MC and wait for MC to be READY + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Check Standalone is configured in MC Config Map + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Add another standalone instance in namespace + testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") + standaloneTwoSpec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("2"), + "memory": resource.MustParse("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("0.2"), + "memory": resource.MustParse("256Mi"), + }, + }, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone two to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Check both standalones are configured in MC Config Map + standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Delete standalone two and ensure MC is updated + testcaseEnvInst.Log.Info("Deleting second standalone deployment from namespace", "Standalone Name", standaloneTwoName) + deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) + err = deployment.DeleteCR(ctx, standaloneTwo) + Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Check standalone one is still configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // Check Standalone Two NOT configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, false) +} + +func verifyStandaloneInMC(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, pods []string, mcName string, shouldExist bool) { + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, pods, "SPLUNK_STANDALONE_URL", mcName, shouldExist) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, pods, mcName, shouldExist, false) +} + +// MCReconfigParams holds the service name and URL parameters that differ between +// V3 (master) and V4 (manager) monitoring console tests. +type MCReconfigParams struct { + CMServiceNameFmt string // format string for CM service name (e.g., testenv.ClusterMasterServiceName) + CMURLKey string // config map URL key (e.g., "SPLUNK_CLUSTER_MASTER_URL" or splcommon.ClusterManagerURL) +} + +// VerifyMCTwoAfterCMReconfig verifies that MC Two is correctly configured after the Cluster Manager +// has been reconfigured to point to it: CM and indexers should be present, SH should be absent. +// If checkDeployerAbsent is true, also verifies deployer is absent on MC Two (used in C3 tests). +func VerifyMCTwoAfterCMReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, checkDeployerAbsent bool) { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + + if checkDeployerAbsent { + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) +} + +// VerifyMCOneAfterCMReconfig verifies that MC One is correctly configured after the Cluster Manager +// has been reconfigured away from it: CM should be absent, SH should still be present. +// If checkDeployerPresent is true, also verifies deployer is still present on MC One (used in M4 tests). +func VerifyMCOneAfterCMReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, checkDeployerPresent bool) { + + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false) + + // CSPL-619: Indexer verification on MC One is commented out in all test variants + + if checkDeployerPresent { + testcaseEnvInst.Log.Info("Verify Deployer still in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + } + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) +} + +// VerifyMCTwoAfterSHCReconfig verifies that MC Two has all components (CM, deployer, SH, indexers) +// after the SHC has been reconfigured to point to it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCTwoAfterSHCReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, timeout time.Duration) { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify Deployer in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig (with wait)") + err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, timeout) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig (with wait)") + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, timeout) + Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") + } else { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + } +} + +// VerifyMCOneAfterSHCReconfig verifies that MC One has lost all components (CM, deployer, SH) +// after the SHC has been reconfigured away from it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCOneAfterSHCReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, timeout time.Duration) { + + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false) + + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig (with wait)") + err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, timeout) + Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") + } else { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) + } + + // CSPL-619: Indexer verification on MC One is commented out in all test variants +} diff --git a/test/secret/manager_secret_c3_test.go b/test/secret/manager_secret_c3_test.go deleted file mode 100644 index 6944755b4..000000000 --- a/test/secret/manager_secret_c3_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_m4_test.go b/test/secret/manager_secret_m4_test.go deleted file mode 100644 index a51dac966..000000000 --- a/test/secret/manager_secret_m4_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_s1_test.go b/test/secret/manager_secret_s1_test.go deleted file mode 100644 index 58a671b67..000000000 --- a/test/secret/manager_secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("managersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("managersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_c3_test.go b/test/secret/secret_c3_test.go deleted file mode 100644 index 8baee517c..000000000 --- a/test/secret/secret_c3_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for ClusterInitialized event to confirm cluster is fully initialized - idxcName := deployment.GetName() + "-idxc" - err = testcaseEnvInst.WaitForClusterInitialized(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterInitialized event on IndexerCluster") - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on SearchHeadCluster - shcName := deployment.GetName() + "-shc" - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on IndexerCluster - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_m4_test.go b/test/secret/secret_m4_test.go deleted file mode 100644 index 3a1efe591..000000000 --- a/test/secret/secret_m4_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_s1_test.go b/test/secret/secret_s1_test.go deleted file mode 100644 index d680a4661..000000000 --- a/test/secret/secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("mastersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("mastersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("mastersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_suite_test.go b/test/secret/secret_suite_test.go index c7d40946a..f12d3ed96 100644 --- a/test/secret/secret_suite_test.go +++ b/test/secret/secret_suite_test.go @@ -15,7 +15,6 @@ package secret import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 10 * ConsistentPollInterval -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "secret-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/secret/secret_test.go b/test/secret/secret_test.go new file mode 100644 index 000000000..c075d3be5 --- /dev/null +++ b/test/secret/secret_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Secret Test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + // S1 tests — both label variants use identical V4 config + s1SecretLabels := []string{"mastersecret", "managersecret"} + + for _, label := range s1SecretLabels { + label := label + Context("Standalone deployment (S1) with LM and MC", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(label+", integration, s1: Secret update on a standalone instance with LM and MC", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteWithMCRefTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // C3 tests — V3 (master) and V4 (manager) variants + c3SecretConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastersecret", testenv.NewClusterReadinessConfigV3}, + {"", "managersecret", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range c3SecretConfigs { + tc := tc + Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", smoke, c3: secret update on indexers and search head cluster", func() { + config := tc.newConfig() + RunC3SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // M4 tests — V3 (master) and V4 (manager) variants + m4SecretConfigs := []struct { + namePrefix string + label string + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastersecret", testenv.NewClusterReadinessConfigV3}, + {"", "managersecret", testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range m4SecretConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { + BeforeEach(func() { + testenv.SpecifiedTestTimeout = 40000 + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration, m4: secret update on multisite indexers and search head cluster", func() { + config := tc.newConfig() + RunM4SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } +}) diff --git a/test/secret/secret_test_shared.go b/test/secret/secret_test_shared.go new file mode 100644 index 000000000..6041cb28f --- /dev/null +++ b/test/secret/secret_test_shared.go @@ -0,0 +1,279 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifySecretsPropagated checks that the given secret data has been propagated to all +// versioned secret objects, pods, server config, input config, and via the API. +func verifySecretsPropagated(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, secretData map[string][]byte, updated bool) { + // Once Pods are READY check each versioned secret for updated secret keys + secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) + + // Verify Secrets on versioned secret objects + testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretData, updated) + + // Once Pods are READY check each pod for updated secret keys + verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) + + // Verify secrets on pods + testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretData, updated) + + // Verify Secrets on ServerConf on Pod + testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretData, updated) + + // Verify Hec token on InputConf on Pod + testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretData, updated) + + // Verify Secrets via api access on Pod + testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretData, updated) +} + +// verifyLMAndStandaloneReady waits for License Manager then Standalone to reach READY status. +func verifyLMAndStandaloneReady(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, standalone *enterpriseApi.Standalone) { + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// verifyLMAndClusterManagerReady waits for License Manager then Cluster Manager to reach READY status. +func verifyLMAndClusterManagerReady(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) +} + +// generateAndApplySecretUpdate creates randomized secret data and applies it to the namespace-scoped +// secret object, returning the updated data map for subsequent verification. +func generateAndApplySecretUpdate(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, namespaceScopedSecretName string) map[string][]byte { + modifiedHecToken := testenv.GetRandomHECToken() + modifiedValue := testenv.RandomDNSName(10) + updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifiedValue, modifiedValue, modifiedValue, modifiedValue) + err := testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) + Expect(err).To(Succeed(), "Unable to update secret Object") + return updatedSecretData +} + +// RunS1SecretUpdateTest runs the standard S1 secret update test workflow +func RunS1SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + // Create standalone Deployment with License Manager + mcRef := deployment.GetName() + var standalone *enterpriseApi.Standalone + var err error + + standalone, err = config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + // Deploy and verify Monitoring Console + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err = testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Update Secret Value on Secret Object + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + + // Ensure standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} + +// RunS1SecretDeleteTest runs the standard S1 secret delete test workflow +func RunS1SecretDeleteTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + // Create standalone Deployment with License Manager + mcRef := deployment.GetName() + var standalone *enterpriseApi.Standalone + var err error + + standalone, err = config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + // Deploy and verify Monitoring Console + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete Secret Object + err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + // Ensure standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretStruct.Data, false) +} + +// RunS1SecretDeleteWithMCRefTest runs the S1 secret delete test with MC reference workflow +func RunS1SecretDeleteWithMCRefTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Create standalone Deployment with MonitoringConsoleRef + var standalone *enterpriseApi.Standalone + var err error + + mcName := deployment.GetName() + standaloneSpec := testenv.NewStandaloneSpecWithMCRef(testcaseEnvInst.GetSplunkImage(), mcName) + standalone, err = deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete Secret Object + err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + // Ensure standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretStruct.Data, false) +} + +// RunC3SecretUpdateTest runs the standard C3 secret update test workflow +func RunC3SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + mcRef := deployment.GetName() + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + testcaseEnvInst.VerifyC3ComponentsReady(ctx, deployment) + + mc, resourceVersion, namespaceScopedSecretName := deployMCAndVerifyInitialSecret(ctx, deployment, testcaseEnvInst) + + // Update Secret Value on Secret Object + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + + config.VerifyClusterManagerPhaseUpdating(ctx, deployment, testcaseEnvInst) + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + + // Ensure Search Head Cluster go to Ready phase + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Wait for PasswordSyncCompleted event on SearchHeadCluster + shcName := deployment.GetName() + "-shc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") + + // Ensure Indexers go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Wait for PasswordSyncCompleted event on IndexerCluster + idxcName := deployment.GetName() + "-idxc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Verify RF SF is met + testcaseEnvInst.Log.Info("Checking RF SF after secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} + +func deployMCAndVerifyInitialSecret(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) (*enterpriseApi.MonitoringConsole, string, string) { + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + testcaseEnvInst.Log.Info("Checking RF SF before secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + return mc, resourceVersion, namespaceScopedSecretName +} + +// RunM4SecretUpdateTest runs the standard M4 secret update test workflow +func RunM4SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Download License File and create config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + siteCount := 3 + mcName := deployment.GetName() + var err error + + err = config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcName) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + mc, resourceVersion, namespaceScopedSecretName := deployMCAndVerifyInitialSecret(ctx, deployment, testcaseEnvInst) + + // Update Secret Value on Secret Object + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + + config.VerifyClusterManagerPhaseUpdating(ctx, deployment, testcaseEnvInst) + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Verify RF SF is met + testcaseEnvInst.Log.Info("Checking RF SF after secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} diff --git a/test/smartstore/manager_smartstore_test.go b/test/smartstore/manager_smartstore_test.go deleted file mode 100644 index f54a98da5..000000000 --- a/test/smartstore/manager_smartstore_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package smartstore - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Smartstore test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("managersmartstore, smoke: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-two" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } - }) - }) -}) diff --git a/test/smartstore/smartstore_suite_test.go b/test/smartstore/smartstore_suite_test.go index 693ca4cf8..112ce28c8 100644 --- a/test/smartstore/smartstore_suite_test.go +++ b/test/smartstore/smartstore_suite_test.go @@ -1,8 +1,20 @@ +// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -10,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smartstore-" + testenv.RandomDNSName(3) @@ -26,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/smartstore/smartstore_test.go b/test/smartstore/smartstore_test.go index bdd100f37..daa4dfc7a 100644 --- a/test/smartstore/smartstore_test.go +++ b/test/smartstore/smartstore_test.go @@ -1,363 +1,97 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "context" - "fmt" "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("Smartstore test", func() { var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() var deployment *testenv.Deployment + ctx := context.TODO() - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - + smartstoreConfigs := []struct { + namePrefix string + label string + s1IndexesTimeout time.Duration + newConfig func() *testenv.ClusterReadinessConfig + }{ + {"master", "mastersmartstore", 2 * time.Minute, testenv.NewClusterReadinessConfigV3}, + {"", "managersmartstore", 5 * time.Minute, testenv.NewClusterReadinessConfigV4}, + } + + for _, tc := range smartstoreConfigs { + tc := tc + Context("Standalone Deployment (S1)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.label+", integration: Can configure multiple indexes through app", func() { + RunS1MultipleIndexesTest(ctx, deployment, testcaseEnvInst, tc.s1IndexesTimeout) + }) + + It(tc.label+", integration: Can configure indexes which use default volumes through app", func() { + RunS1DefaultVolumesTest(ctx, deployment, testcaseEnvInst) + }) }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("mastersmartstore, m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - oldBundleHash := testenv.GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.namePrefix) + }) - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify new bundle is pushed - testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } + It(tc.label+", m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { + config := tc.newConfig() + RunM4MultisiteSmartStoreTest(ctx, deployment, testcaseEnvInst, config) + }) }) - }) + } Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Etc storage", func() { - - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Etc and pvc for Var volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - EtcVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageEtcEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Var storage", func() { - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Var and pvc for Etc volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - VarVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageVarEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Etc storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageEtcEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, true) + }) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Var storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageVarEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, false) }) }) }) diff --git a/test/smartstore/smartstore_test_shared.go b/test/smartstore/smartstore_test_shared.go new file mode 100644 index 000000000..ee501a7b2 --- /dev/null +++ b/test/smartstore/smartstore_test_shared.go @@ -0,0 +1,228 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package smartstore + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" +) + +// RunS1MultipleIndexesTest runs the standard S1 multiple indexes SmartStore test workflow +func RunS1MultipleIndexesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, waitTimeout time.Duration) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexVolumeMap := map[string]string{ + "test-index-" + testenv.RandomDNSName(3): volName, + "test-index-" + testenv.RandomDNSName(3): volName, + } + testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) + + var indexSpec []enterpriseApi.IndexSpec + volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + // Create index volume spec from index volume map + for index, volume := range indexVolumeMap { + indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) + } + + // Generate smartstore spec + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volumeSpec, + IndexList: indexSpec, + } + + // Deploy Standalone + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Wait for Standalone to reach Ready phase + err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, waitTimeout) + Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") + + // Verify standalone goes to ready state and stays ready + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + for indexName := range indexVolumeMap { + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + } + + // Ingest data to the index + for indexName := range indexVolumeMap { + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } + + // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 + for indexName := range indexVolumeMap { + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + } +} + +// RunS1DefaultVolumesTest runs the standard S1 default volumes SmartStore test workflow +func RunS1DefaultVolumesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} + defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} + cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} + + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + Defaults: defaultSmartStoreSpec, + CacheManagerConf: cacheManagerSmartStoreSpec, + } + + // Deploy Standalone with given smartstore spec + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Verify standalone goes to ready state + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + + // Check special index configs + testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) + + // Ingest data to the index + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + + // Roll Hot Buckets on the test index by restarting splunk + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + + // Check for indexes on S3 + testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + + // Verify Cachemanager Values + serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" + + // Validate MaxCacheSizeMB + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) + + // Validate EvictionPaddingSizeMB + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) + + // Validate MaxConcurrentDownloads + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) + + // Validate MaxConcurrentUploads + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) + + // Validate EvictionPolicy + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) +} + +// RunS1EphemeralStorageTest deploys a Standalone with one ephemeral storage volume configured and verifies it is ready. +// Pass etcStorage=true to set EtcVolumeStorageConfig, false to set VarVolumeStorageConfig. +func RunS1EphemeralStorageTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, storageConfig enterpriseApi.StorageClassSpec, etcStorage bool) { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + } + if etcStorage { + spec.CommonSplunkSpec.EtcVolumeStorageConfig = storageConfig + } else { + spec.CommonSplunkSpec.VarVolumeStorageConfig = storageConfig + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// verifyM4ClusterAndRFSF verifies cluster manager and multisite cluster are ready and RF/SF is met. +func verifyM4ClusterAndRFSF(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, siteCount int) { + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyMultisiteClusterReadyAndRFSF(ctx, deployment, siteCount) +} + +// RunM4MultisiteSmartStoreTest runs the standard M4 multisite SmartStore test workflow +func RunM4MultisiteSmartStoreTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + } + + siteCount := 3 + var err error + + err = config.DeployMultisiteClusterWithIndexes(ctx, deployment, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + verifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount) + + // Use multisite workflow helper to verify index, ingest data, roll to warm, and verify on S3 + testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, deployment.GetName(), siteCount, indexName, 2000) + + // Get old bundle hash before adding new index + oldBundleHash := config.GetBundleHash(ctx, deployment) + + testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") + indexNameTwo := "test-index-" + testenv.RandomDNSName(3) + indexList := []string{indexName, indexNameTwo} + newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} + + // Update CR with new index based on API version + config.AppendSmartStoreIndex(ctx, deployment, newIndex) + + verifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount) + + // Verify new bundle is pushed (only for v3) + if config.APIVersion == "v3" { + testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) + } + + // Verify both indexes on all sites + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + for _, index := range indexList { + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) + } + } + + // Use multisite workflow helper for the new index + testcaseEnvInst.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) + testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, deployment.GetName(), siteCount, indexNameTwo, 2000) +} diff --git a/test/smoke/cluster_manager_sites_response.go b/test/smoke/cluster_manager_sites_response.go deleted file mode 100644 index f9bcdee60..000000000 --- a/test/smoke/cluster_manager_sites_response.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package smoke - -// ClusterManagerSitesResponse is a representation of the sites managed by a Splunk cluster-manager -// Endpoint: /services/cluster/manager/sites -type ClusterManagerSitesResponse struct { - Entries []ClusterManagerSitesEntry `json:"entry"` -} - -// ClusterManagerSitesEntry represents a site of an indexer cluster with its metadata -type ClusterManagerSitesEntry struct { - Name string `json:"name"` - Content ClusterManagerSitesContent `json:"content"` -} - -// ClusterManagerSitesContent represents detailed information about a site -type ClusterManagerSitesContent struct { - Peers map[string]ClusterManagerSitesPeer `json:"peers"` -} - -// ClusterManagerSitesPeer reprensents an indexer peer member of a site -type ClusterManagerSitesPeer struct { - ServerName string `json:"server_name"` -} diff --git a/test/smoke/smoke_suite_test.go b/test/smoke/smoke_suite_test.go index b47f8a984..de4c35a89 100644 --- a/test/smoke/smoke_suite_test.go +++ b/test/smoke/smoke_suite_test.go @@ -15,7 +15,6 @@ package smoke import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smoke-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index d6d969819..7823ecea7 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -15,16 +15,10 @@ package smoke import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("Smoke test", func() { @@ -34,134 +28,41 @@ var _ = Describe("Smoke test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1)", func() { It("smoke, basic, s1: can deploy a standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + testenv.RunStandaloneDeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName()) }) }) Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { It("smoke, basic, c3: can deploy indexers and search head cluster", func() { - - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunC3DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 3, "") }) }) Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { It("smoke, basic, m4: can deploy indexers and search head cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunM4DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 1, 3, "") }) }) Context("Multisite cluster deployment (M1 - multisite indexer cluster)", func() { It("smoke, basic: can deploy multisite indexers cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteCluster(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunM1DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 1, 3) }) }) Context("Standalone deployment (S1) with Service Account", func() { It("smoke, basic, s1: can deploy a standalone instance attached to a service account", func() { - // Create Service Account serviceAccountName := "smoke-service-account" - testcaseEnvInst.CreateServiceAccount(serviceAccountName) - - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ServiceAccount: serviceAccountName, - }, - } - - // Create standalone Deployment with License Manager - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify serviceAccount is configured on Pod - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + testenv.RunStandaloneWithServiceAccountWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), serviceAccountName) }) }) }) diff --git a/test/testenv/assertion_helpers.go b/test/testenv/assertion_helpers.go new file mode 100644 index 000000000..9b1574951 --- /dev/null +++ b/test/testenv/assertion_helpers.go @@ -0,0 +1,201 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// ScaleSearchHeadCluster scales a Search Head Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleSearchHeadCluster(ctx context.Context, deployment *Deployment, deploymentName string, newReplicas int) { + shcName := deploymentName + "-shc" + + // Get instance of current SHC CR with latest config + shc := &enterpriseApi.SearchHeadCluster{} + GetInstanceWithExpect(ctx, deployment, shc, shcName, "Failed to get instance of Search Head Cluster") + + // Update Replicas of SHC + shc.Spec.Replicas = int32(newReplicas) + UpdateCRWithExpect(ctx, deployment, shc, "Failed to scale Search Head Cluster") + + // Verify Search Head cluster scales up and goes to ScalingUp phase + testcaseenv.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) +} + +// ScaleIndexerCluster scales an Indexer Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleIndexerCluster(ctx context.Context, deployment *Deployment, deploymentName string, newReplicas int) { + idxcName := deploymentName + "-idxc" + + // Get instance of current Indexer CR with latest config + idxc := &enterpriseApi.IndexerCluster{} + GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(newReplicas) + UpdateCRWithExpect(ctx, deployment, idxc, "Failed to scale Indexer Cluster") + + // Verify Indexer cluster scales up and goes to ScalingUp phase + testcaseenv.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) +} + +// UpdateMonitoringConsoleRef updates the MonitoringConsoleRef in a CR and waits for the change to apply +func UpdateMonitoringConsoleRefAndVerify(ctx context.Context, deployment *Deployment, testcaseenv *TestCaseEnv, obj interface{}, instanceName string, newMCName string) { + // Get current resource version before update + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, obj) + + // Update the MonitoringConsoleRef based on the type + switch cr := obj.(type) { + case *enterpriseApi.ClusterManager: + GetInstanceWithExpect(ctx, deployment, cr, instanceName, "Failed to get instance") + cr.Spec.MonitoringConsoleRef.Name = newMCName + UpdateCRWithExpect(ctx, deployment, cr, "Failed to update MonitoringConsoleRef") + case *enterpriseApi.SearchHeadCluster: + GetInstanceWithExpect(ctx, deployment, cr, instanceName, "Failed to get instance") + cr.Spec.MonitoringConsoleRef.Name = newMCName + UpdateCRWithExpect(ctx, deployment, cr, "Failed to update MonitoringConsoleRef") + } + + // Wait for custom resource version to change + testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, obj, resourceVersion) +} + +// VerifyMCConfigForC3Cluster verifies the standard MC configuration for a C3 cluster +func (testcaseenv *TestCaseEnv) VerifyMCConfigForC3Cluster(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shReplicas int, indexerReplicas int, shouldExist bool) { + // Check Cluster Manager in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(ClusterMasterServiceName, deploymentName)}, "SPLUNK_CLUSTER_MASTER_URL", mcName, shouldExist) + + // Check Deployer in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(DeployerServiceName, deploymentName)}, "SPLUNK_DEPLOYER_URL", mcName, shouldExist) + + // Check Search Head Pods in MC Config Map + shPods := GeneratePodNameSlice(SearchHeadPod, deploymentName, shReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, shouldExist) + + // Check Search Heads in MC Pod config string + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, shouldExist, false) + + // Check Indexers in MC Pod config string + indexerPods := GeneratePodNameSlice(IndexerPod, deploymentName, indexerReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, shouldExist, true) +} + +// VerifyMCConfigForM4Cluster verifies the standard MC configuration for an M4 multisite cluster +func (testcaseenv *TestCaseEnv) VerifyMCConfigForM4Cluster(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shReplicas int, indexerReplicas int, siteCount int, shouldExist bool) { + // Check Cluster Manager in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(ClusterMasterServiceName, deploymentName)}, "SPLUNK_CLUSTER_MASTER_URL", mcName, shouldExist) + + // Check Deployer in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(DeployerServiceName, deploymentName)}, "SPLUNK_DEPLOYER_URL", mcName, shouldExist) + + // Check Search Head Pods in MC Config Map + shPods := GeneratePodNameSlice(SearchHeadPod, deploymentName, shReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, shouldExist) + + // Check Search Heads in MC Pod config string + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, shouldExist, false) + + // Check Indexers in MC Pod config string + indexerPods := GeneratePodNameSlice(MultiSiteIndexerPod, deploymentName, indexerReplicas, true, siteCount) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, shouldExist, true) +} + +// DeployAndVerifyC3WithMC deploys a C3 cluster with a given MC and verifies all components are ready +func (testcaseenv *TestCaseEnv) DeployAndVerifyC3WithMC(ctx context.Context, deployment *Deployment, deploymentName string, indexerReplicas int, mcName string) { + err := deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deploymentName, indexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + // Verify all components are ready + testcaseenv.VerifyClusterMasterReady(ctx, deployment) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) + testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// DeployStandaloneWithMCRef deploys a standalone instance with a MonitoringConsoleRef +func (testcaseenv *TestCaseEnv) DeployStandaloneWithMCRef(ctx context.Context, deployment *Deployment, deploymentName string, mcName string) *enterpriseApi.Standalone { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseenv.GetSplunkImage(), + }, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deploymentName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for Standalone to be in READY status + testcaseenv.VerifyStandaloneReady(ctx, deployment, deploymentName, standalone) + + return standalone +} + +// VerifyStandaloneInMC verifies that a standalone instance is configured in the MC +func (testcaseenv *TestCaseEnv) VerifyStandaloneInMC(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shouldExist bool) { + standalonePod := fmt.Sprintf(StandalonePod, deploymentName, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{standalonePod}, "SPLUNK_STANDALONE_URL", mcName, shouldExist) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, []string{standalonePod}, mcName, shouldExist, false) +} + +// VerifyLMConfiguredOnIndexers verifies License Manager is configured on all indexer pods +func VerifyLMConfiguredOnIndexers(ctx context.Context, deployment *Deployment, deploymentName string, indexerCount int) { + for i := 0; i < indexerCount; i++ { + indexerPodName := fmt.Sprintf(IndexerPod, deploymentName, i) + VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) + } +} + +// VerifyLMConfiguredOnSearchHeads verifies License Manager is configured on all search head pods +func VerifyLMConfiguredOnSearchHeads(ctx context.Context, deployment *Deployment, deploymentName string, searchHeadCount int) { + for i := 0; i < searchHeadCount; i++ { + searchHeadPodName := fmt.Sprintf(SearchHeadPod, deploymentName, i) + VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) + } +} + +// VerifyLMConfiguredOnMultisiteIndexers verifies License Manager is configured on all multisite indexer pods +func VerifyLMConfiguredOnMultisiteIndexers(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int) { + for i := 1; i <= siteCount; i++ { + indexerPodName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, i, 0) + VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) + } +} + +// IngestDataOnIndexers ingests test data on all indexer pods +func IngestDataOnIndexers(ctx context.Context, deployment *Deployment, deploymentName string, indexerCount int, indexName string, logLineCount int) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deploymentName, i) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, logLineCount) + IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } +} + +// IngestDataOnMultisiteIndexers ingests test data on all multisite indexer pods +func IngestDataOnMultisiteIndexers(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string, logLineCount int) { + for site := 1; site <= siteCount; site++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, site, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, logLineCount) + IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } +} diff --git a/test/testenv/cmutil.go b/test/testenv/cmutil.go index a1b71f3ce..68e0bb499 100644 --- a/test/testenv/cmutil.go +++ b/test/testenv/cmutil.go @@ -44,7 +44,7 @@ type ClusterManagerSitesContent struct { Peers map[string]ClusterManagerSitesPeer `json:"peers"` } -// ClusterManagerSitesPeer reprensents an indexer peer member of a site +// ClusterManagerSitesPeer represents an indexer peer member of a site type ClusterManagerSitesPeer struct { ServerName string `json:"server_name"` } @@ -194,21 +194,6 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { return searchHeadRemoved } -// RollHotBuckets roll hot buckets in cluster -func RollHotBuckets(ctx context.Context, deployment *Deployment) bool { - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - stdin := "/opt/splunk/bin/splunk rolling-restart cluster-peers -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - - return strings.Contains(stdout, "Rolling restart of all cluster peers has been initiated.") -} - // ClusterManagerInfoEndpointResponse is represtentation of /services/cluster/manager/info endpoint type ClusterManagerInfoEndpointResponse struct { Entry []struct { diff --git a/test/testenv/common_test_patterns.go b/test/testenv/common_test_patterns.go new file mode 100644 index 000000000..35e561eda --- /dev/null +++ b/test/testenv/common_test_patterns.go @@ -0,0 +1,278 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + corev1 "k8s.io/api/core/v1" +) + +// ClusterReadinessConfig holds v3/v4 API version callbacks for cluster and license manager +// readiness verification. Shared across test packages to avoid per-package duplication. +type ClusterReadinessConfig struct { + LicenseManagerReady func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) + ClusterManagerReady func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) + APIVersion string +} + +// NewClusterReadinessConfigV3 creates a ClusterReadinessConfig for v3 API (LicenseMaster/ClusterMaster) +func NewClusterReadinessConfigV3() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ + LicenseManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) + }, + ClusterManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyClusterMasterReady(ctx, deployment) + }, + APIVersion: "v3", + } +} + +// NewClusterReadinessConfigV4 creates a ClusterReadinessConfig for v4 API (LicenseManager/ClusterManager) +func NewClusterReadinessConfigV4() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ + LicenseManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) + }, + ClusterManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyClusterManagerReady(ctx, deployment) + }, + APIVersion: "v4", + } +} + +// DeployStandaloneWithLM deploys a standalone with the appropriate License Manager type for +// the API version: LicenseMaster (v3) or LicenseManager (v4). +func (c *ClusterReadinessConfig) DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) { + if c.APIVersion == "v3" { + return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) + } + return deployment.DeployStandaloneWithLM(ctx, name, mcRef) +} + +// DeployMultisiteCluster deploys a multisite cluster with the appropriate Cluster Manager type +// for the API version: ClusterMaster (v3) or ClusterManager (v4). +func (c *ClusterReadinessConfig) DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error { + if c.APIVersion == "v3" { + return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) + } + return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) +} + +// VerifyClusterManagerPhaseUpdating asserts the Cluster Manager (or ClusterMaster for v3) +// has entered the Updating phase. +func (c *ClusterReadinessConfig) VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + if c.APIVersion == "v3" { + testcaseEnv.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + } else { + testcaseEnv.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + } +} + +// ClusterManagerPVCType returns the PVC label fragment for the Cluster Manager: +// "cluster-master" for v3, "cluster-manager" for v4. +func (c *ClusterReadinessConfig) ClusterManagerPVCType() string { + if c.APIVersion == "v3" { + return "cluster-master" + } + return "cluster-manager" +} + +// DeleteClusterManager fetches and deletes the Cluster Manager CR for the appropriate API version. +func (c *ClusterReadinessConfig) DeleteClusterManager(ctx context.Context, deployment *Deployment) { + name := deployment.GetName() + if c.APIVersion == "v3" { + cm := &enterpriseApiV3.ClusterMaster{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Unable to GET Cluster Master instance", "Cluster Master Name", cm) + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Master instance", "Cluster Master Name", cm) + } else { + cm := &enterpriseApi.ClusterManager{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manager Name", cm) + } +} + +// DeployMultisiteClusterWithIndexes deploys a multisite cluster with SmartStore indexes using +// the appropriate Cluster Manager type for the API version. +func (c *ClusterReadinessConfig) DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { + if c.APIVersion == "v3" { + return deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) + } + return deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) +} + +// GetBundleHash returns the current bundle hash for the Cluster Manager (or ClusterMaster for v3). +func (c *ClusterReadinessConfig) GetBundleHash(ctx context.Context, deployment *Deployment) string { + if c.APIVersion == "v3" { + return GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") + } + return GetClusterManagerBundleHash(ctx, deployment, "ClusterManager") +} + +// AppendSmartStoreIndex appends a new SmartStore index to the Cluster Manager CR +// for the appropriate API version. +func (c *ClusterReadinessConfig) AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) { + name := deployment.GetName() + if c.APIVersion == "v3" { + cm := &enterpriseApiV3.ClusterMaster{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed to add new index to cluster master") + } else { + cm := &enterpriseApi.ClusterManager{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed to add new index to cluster manager") + } +} + +// DeployMCAndGetVersion deploys and verifies a Monitoring Console, then returns both the MC +// instance and its current resource version. +func (testcaseenv *TestCaseEnv) DeployMCAndGetVersion(ctx context.Context, deployment *Deployment, name string, lmRef string) (*enterpriseApi.MonitoringConsole, string) { + mc := testcaseenv.DeployAndVerifyMonitoringConsole(ctx, deployment, name, lmRef) + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, mc) + return mc, resourceVersion +} + +// DeployAndVerifyStandalone deploys a standalone instance and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyStandalone(ctx context.Context, deployment *Deployment, name string, mcRef string, licenseManagerRef string) *enterpriseApi.Standalone { + standalone, err := deployment.DeployStandalone(ctx, name, mcRef, licenseManagerRef) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + testcaseenv.VerifyStandaloneReady(ctx, deployment, name, standalone) + return standalone +} + +// DeployAndVerifyMonitoringConsole deploys a monitoring console and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyMonitoringConsole(ctx context.Context, deployment *Deployment, name string, licenseManagerRef string) *enterpriseApi.MonitoringConsole { + mc, err := deployment.DeployMonitoringConsole(ctx, name, licenseManagerRef) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") + + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, name, mc) + return mc +} + +// VerifyIndexerCPULimits verifies CPU limits on all indexer pods in a single-site cluster +func (testcaseenv *TestCaseEnv) VerifyIndexerCPULimits(deployment *Deployment, deploymentName string, indexerCount int, expectedCPULimit string) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deploymentName, i) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// VerifySearchHeadCPULimits verifies CPU limits on all search head pods +func (testcaseenv *TestCaseEnv) VerifySearchHeadCPULimits(deployment *Deployment, deploymentName string, searchHeadCount int, expectedCPULimit string) { + for i := 0; i < searchHeadCount; i++ { + podName := fmt.Sprintf(SearchHeadPod, deploymentName, i) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// VerifyC3ComponentsReady verifies SHC and single-site indexers are ready (without CM check or RFSF). +func (testcaseenv *TestCaseEnv) VerifyC3ComponentsReady(ctx context.Context, deployment *Deployment) { + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) + testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// VerifyM4ComponentsReady verifies multisite indexers, multisite status, and SHC are ready (without CM check or RFSF). +func (testcaseenv *TestCaseEnv) VerifyM4ComponentsReady(ctx context.Context, deployment *Deployment, siteCount int) { + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func (testcaseenv *TestCaseEnv) VerifyMCVersionChangedAndReady(ctx context.Context, deployment *Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) { + testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} + +// VerifyClusterReadyAndRFSF is a common verification pattern that checks cluster is ready and RF/SF is met +func (testcaseenv *TestCaseEnv) VerifyClusterReadyAndRFSF(ctx context.Context, deployment *Deployment) { + testcaseenv.VerifyClusterManagerReady(ctx, deployment) + testcaseenv.VerifyC3ComponentsReady(ctx, deployment) + testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// VerifyMultisiteClusterReadyAndRFSF is a common verification pattern for multisite clusters +func (testcaseenv *TestCaseEnv) VerifyMultisiteClusterReadyAndRFSF(ctx context.Context, deployment *Deployment, siteCount int) { + testcaseenv.VerifyClusterManagerReady(ctx, deployment) + testcaseenv.VerifyM4ComponentsReady(ctx, deployment, siteCount) + testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// TriggerAndVerifyTelemetry is a common pattern for telemetry verification +func (testcaseenv *TestCaseEnv) TriggerAndVerifyTelemetry(ctx context.Context, deployment *Deployment, prevSubmissionTime string) { + testcaseenv.TriggerTelemetrySubmission(ctx, deployment) + testcaseenv.VerifyTelemetry(ctx, deployment, prevSubmissionTime) +} + +// VerifyProbeConfigAndScripts verifies probe config map exists and probe scripts are present on all pods. +// If includeStartup is true, the startup probe script is also checked. +func (testcaseenv *TestCaseEnv) VerifyProbeConfigAndScripts(ctx context.Context, deployment *Deployment, includeStartup bool) { + testcaseenv.Log.Info("Get config map for livenessProbe and readinessProbe") + configMapName := enterprise.GetProbeConfigMapName(testcaseenv.GetName()) + _, err := GetConfigMap(ctx, deployment, testcaseenv.GetName(), configMapName) + Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", configMapName) + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} + if includeStartup { + scriptsNames = append(scriptsNames, enterprise.GetStartupScriptName()) + } + allPods := DumpGetPods(testcaseenv.GetName()) + testcaseenv.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) +} + +// NewStandaloneSpecWithMCRef creates a StandaloneSpec with a MonitoringConsoleRef set to the given MC name. +func NewStandaloneSpecWithMCRef(image string, mcName string) enterpriseApi.StandaloneSpec { + return enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: image, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } +} + +// VerifyLMConfiguredOnMC verifies that the License Manager is configured on the Monitoring Console pod. +func VerifyLMConfiguredOnMC(ctx context.Context, deployment *Deployment) { + monitoringConsolePodName := fmt.Sprintf(MonitoringConsolePod, deployment.GetName()) + VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) +} + +// StandardC3Verification performs the standard set of verifications for a C3 cluster +// This includes cluster ready, RF/SF met, and monitoring console ready +func (testcaseenv *TestCaseEnv) StandardC3Verification(ctx context.Context, deployment *Deployment, mcName string, mc *enterpriseApi.MonitoringConsole) { + testcaseenv.VerifyClusterReadyAndRFSF(ctx, deployment) + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) +} diff --git a/test/testenv/cr_update_helpers.go b/test/testenv/cr_update_helpers.go new file mode 100644 index 000000000..492b26187 --- /dev/null +++ b/test/testenv/cr_update_helpers.go @@ -0,0 +1,34 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetInstanceWithExpect is a wrapper around GetInstance that includes Gomega expectations +func GetInstanceWithExpect(ctx context.Context, deployment *Deployment, obj client.Object, instanceName string, errorMsg string) { + err := deployment.GetInstance(ctx, instanceName, obj) + Expect(err).To(Succeed(), errorMsg, "Instance Name", instanceName) +} + +// UpdateCRWithExpect is a wrapper around UpdateCR that includes Gomega expectations +func UpdateCRWithExpect(ctx context.Context, deployment *Deployment, obj client.Object, errorMsg string) { + err := deployment.UpdateCR(ctx, obj) + Expect(err).To(Succeed(), errorMsg) +} diff --git a/test/testenv/lmutil.go b/test/testenv/lmutil.go index 9f7aa5dd7..326129786 100644 --- a/test/testenv/lmutil.go +++ b/test/testenv/lmutil.go @@ -1,18 +1,16 @@ // Copyright (c) 2018-2022 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package testenv import ( diff --git a/test/testenv/mcutil.go b/test/testenv/mcutil.go index 36250ae96..10bbaa0b9 100644 --- a/test/testenv/mcutil.go +++ b/test/testenv/mcutil.go @@ -75,7 +75,7 @@ func CheckMCPodReady(ns string) bool { return stsReady && podReady } -// GetConfiguredPeers get list of Peers Configured on Montioring Console +// GetConfiguredPeers get list of Peers Configured on Monitoring Console func GetConfiguredPeers(ns string, mcName string) []string { podName := fmt.Sprintf(MonitoringConsolePod, mcName) var peerList []string @@ -104,18 +104,6 @@ func GetConfiguredPeers(ns string, mcName string) []string { return peerList } -// DeleteMCPod delete monitoring console deployment -func DeleteMCPod(ns string) { - mcSts := fmt.Sprintf(MonitoringConsoleSts, ns) - output, err := exec.Command("kubectl", "delete", "sts", "-n", ns, mcSts).Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete sts -n %s %s", ns, mcSts) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("Monitoring Console Stateful Set deleted", "Statefulset", mcSts, "stdout", output) - } -} - // CheckPodNameOnMC Check given pod is configured on Monitoring console pod func CheckPodNameOnMC(ns string, mcName string, podName string) bool { // Get Peers configured on Monitoring Console diff --git a/test/testenv/multisite_helpers.go b/test/testenv/multisite_helpers.go new file mode 100644 index 000000000..4429dbf02 --- /dev/null +++ b/test/testenv/multisite_helpers.go @@ -0,0 +1,78 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" +) + +// VerifyIndexOnAllSites verifies that an index exists on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + } +} + +// IngestDataOnAllSites ingests data to an index on all indexer pods across all sites +func IngestDataOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string, logLineCount int) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, logLineCount) + IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } +} + +// RollHotToWarmOnAllSites rolls hot buckets to warm on all indexer pods across all sites +func RollHotToWarmOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + RollHotToWarm(ctx, deployment, podName, indexName) + } +} + +// VerifyIndexOnS3AllSites verifies that an index exists on S3 for all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnS3AllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + } +} + +// VerifyCPULimitsOnAllSites verifies CPU limits on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyCPULimitsOnAllSites(deployment *Deployment, deploymentName string, siteCount int, expectedCPULimit string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// MultisiteIndexerWorkflow encapsulates the common workflow for multisite indexer operations: +// verify index, ingest data, roll to warm, verify on S3 +func (testcaseenv *TestCaseEnv) MultisiteIndexerWorkflow(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string, logLineCount int) { + // Verify index exists on all sites + testcaseenv.VerifyIndexOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Ingest data on all sites + IngestDataOnAllSites(ctx, deployment, deploymentName, siteCount, indexName, logLineCount) + + // Roll hot to warm on all sites + RollHotToWarmOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Verify index on S3 for all sites + testcaseenv.VerifyIndexOnS3AllSites(ctx, deployment, deploymentName, siteCount, indexName) +} diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index 551a377d0..e093e5008 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -60,19 +60,6 @@ func GetIndexOnPod(ctx context.Context, deployment *Deployment, podName string, return indexFound, indexData } -// RestartSplunk Restart splunk inside the container -func RestartSplunk(ctx context.Context, deployment *Deployment, podName string) bool { - stdin := "/opt/splunk/bin/splunk restart -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - return true -} - // RollHotToWarm rolls hot buckets to warm for a given index and pod func RollHotToWarm(ctx context.Context, deployment *Deployment, podName string, indexName string) bool { stdin := "/opt/splunk/bin/splunk _internal call /data/indexes/" + indexName + "/roll-hot-buckets admin:$(cat /mnt/splunk-secrets/password)" diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index 445e395f1..e9559bf5b 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -32,11 +32,6 @@ var ( s3deleteWaitTime = 60 ) -// GetSmartStoreIndexesBucet returns smartstore test bucket name -func GetSmartStoreIndexesBucet() string { - return testIndexesS3Bucket -} - // GetDefaultS3Region returns default AWS Region func GetDefaultS3Region() string { return s3Region diff --git a/test/testenv/search_head_cluster_utils.go b/test/testenv/search_head_cluster_utils.go index 012064501..90c847afb 100644 --- a/test/testenv/search_head_cluster_utils.go +++ b/test/testenv/search_head_cluster_utils.go @@ -18,39 +18,11 @@ package testenv import ( "context" "fmt" - "os/exec" "strings" logf "sigs.k8s.io/controller-runtime/pkg/log" ) -// DeleteSHC delete Search Head Cluster in given namespace -func DeleteSHC(ns string) { - output, err := exec.Command("kubectl", "delete", "shc", "-n", ns, "--all").Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete shc -n %s --all", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("SHC deleted", "Namespace", ns, "stdout", output) - } -} - -// SHCInNamespace returns true if SHC is present in namespace -func SHCInNamespace(ns string) bool { - output, err := exec.Command("kubectl", "get", "searchheadcluster", "-n", ns).Output() - deleted := true - if err != nil { - cmd := fmt.Sprintf("kubectl get shc -n %s", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - return deleted - } - logf.Log.Info("Output of command", "Output", string(output)) - if strings.Contains(string(output), "No resources found in default namespace") { - deleted = false - } - return deleted -} - // DeployerAppChecksum Get the checksum for each app on the deployer func DeployerAppChecksum(ctx context.Context, deployment *Deployment) map[string]string { appChecksum := make(map[string]string) diff --git a/test/testenv/secretutil.go b/test/testenv/secretutil.go index 1b9ee9ff1..a2f2e09a6 100644 --- a/test/testenv/secretutil.go +++ b/test/testenv/secretutil.go @@ -89,8 +89,8 @@ func GetMountedKey(ctx context.Context, deployment *Deployment, podName string, return stdout } -// GetRandomeHECToken generates a random HEC token -func GetRandomeHECToken() string { +// GetRandomHECToken generates a random HEC token +func GetRandomHECToken() string { return fmt.Sprintf("%s-%s-%s-%s-%s", strings.ToUpper(RandomDNSName(8)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(12))) } diff --git a/test/testenv/test_setup_helpers.go b/test/testenv/test_setup_helpers.go new file mode 100644 index 000000000..4d3685ee5 --- /dev/null +++ b/test/testenv/test_setup_helpers.go @@ -0,0 +1,223 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/joho/godotenv" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + . "github.com/onsi/gomega" +) + +// SetupTestCaseEnv creates a new test case environment and deployment for use in BeforeEach blocks. +// It also validates test prerequisites immediately to fail fast before any long operations. +func SetupTestCaseEnv(testenvInstance *TestEnv, namePrefix string) (*TestCaseEnv, *Deployment) { + name := fmt.Sprintf("%s-%s", namePrefix+testenvInstance.GetName(), RandomDNSName(3)) + testcaseEnvInst, err := NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) + Expect(err).To(Succeed(), "Unable to create testcaseenv") + + deployment, err := testcaseEnvInst.NewDeployment(RandomDNSName(3)) + Expect(err).To(Succeed(), "Unable to create deployment") + + err = testcaseEnvInst.ValidateTestPrerequisites(context.TODO(), deployment) + Expect(err).To(Succeed(), "Test prerequisites validation failed") + + return testcaseEnvInst, deployment +} + +// TeardownTestCaseEnv handles the common teardown logic for test case environments. +func TeardownTestCaseEnv(testcaseEnvInst *TestCaseEnv, deployment *Deployment) { + if types.SpecState(ginkgo.CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil { + testcaseEnvInst.SkipTeardown = true + } + } + + if deployment != nil { + deployment.Teardown() + } + + if testcaseEnvInst != nil { + Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) + } +} + +// CleanupOperatorFile deletes the test_file.img from the operator pod's app download directory +// if filePresentOnOperator is true. +func CleanupOperatorFile(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, filePresentOnOperator bool) { + if filePresentOnOperator { + opPod := GetOperatorPodName(testcaseEnvInst) + podDownloadPath := filepath.Join(AppDownloadVolume, "test_file.img") + DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + } +} + +// TeardownAppFrameworkTestCaseEnv handles teardown for app framework tests with provider-specific +// cloud storage cleanup. cloudCleanup is called only if SkipTeardown is false. +func TeardownAppFrameworkTestCaseEnv(ctx context.Context, testcaseEnvInst *TestCaseEnv, deployment *Deployment, cloudCleanup func(), filePresentOnOperator bool) { + TeardownTestCaseEnv(testcaseEnvInst, deployment) + + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown && cloudCleanup != nil { + cloudCleanup() + } + + CleanupOperatorFile(ctx, deployment, testcaseEnvInst, filePresentOnOperator) +} + +// LoadEnvFile traverses up the directory tree from the current working directory +// to find and load a .env file using godotenv. Returns nil if no .env file is found. +func LoadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } +} + +// SetupS3AppsSuite initialises the test environment and, when running on EKS, +// downloads the V1 and V2 app sets from S3. +func SetupS3AppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "eks" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + err = DownloadFilesFromS3(testDataBucket, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + err = DownloadFilesFromS3(testDataBucket, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// CleanupLocalAppDownloads tears down the test environment and removes locally +// downloaded app directories after a suite run. +func CleanupLocalAppDownloads(testenvInst *TestEnv, dirs ...string) { + if testenvInst != nil { + Expect(testenvInst.Teardown()).ToNot(HaveOccurred()) + } + for _, dir := range dirs { + Expect(os.RemoveAll(dir)).To(Succeed(), "Unable to delete locally downloaded app files from "+dir) + } +} + +// SetupAzureAppsSuite initialises the test environment and, when running on Azure, +// downloads the V1 and V2 app sets from Azure Blob. +func SetupAzureAppsSuite(suiteName, downloadDirV1, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "azure" { + ctx := context.TODO() + + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + containerName := "/test-data/appframework/v1apps/" + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV1, containerName, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + containerName = "/test-data/appframework/v2apps/" + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV2, containerName, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// SetupGCPAppsSuite initialises the test environment and, when running on GCP, +// downloads the V1 and V2 app sets from GCS. +func SetupGCPAppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "gcp" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + testenvInst.Log.Info("logging download details", "bucket", testDataBucket, "appDirV1", appDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) + err = DownloadFilesFromGCP(testDataBucket, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + err = DownloadFilesFromGCP(testDataBucket, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// SetupLicenseConfigMap downloads the license file from the appropriate cloud provider +// and creates a license config map. +func SetupLicenseConfigMap(ctx context.Context, testcaseEnvInst *TestCaseEnv) { + downloadDir := "licenseFolder" + var licenseFilePath string + var err error + + switch ClusterProvider { + case "eks": + licenseFilePath, err = DownloadLicenseFromS3Bucket() + Expect(err).To(Succeed(), "Unable to download license file from S3") + case "azure": + licenseFilePath, err = DownloadLicenseFromAzure(ctx, downloadDir) + Expect(err).To(Succeed(), "Unable to download license file from Azure") + case "gcp": + licenseFilePath, err = DownloadLicenseFromGCPBucket() + Expect(err).To(Succeed(), "Unable to download license file from GCP") + default: + testcaseEnvInst.Log.Info("Skipping license download", "ClusterProvider", ClusterProvider) + return + } + + testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) +} diff --git a/test/testenv/test_workflows.go b/test/testenv/test_workflows.go new file mode 100644 index 000000000..2b095f605 --- /dev/null +++ b/test/testenv/test_workflows.go @@ -0,0 +1,145 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// WorkflowResult contains the result of a workflow execution +type WorkflowResult struct { + Standalone *enterpriseApi.Standalone + ClusterManager *enterpriseApi.ClusterManager + IndexerCluster *enterpriseApi.IndexerCluster + SearchHeadCluster *enterpriseApi.SearchHeadCluster + MonitoringConsole *enterpriseApi.MonitoringConsole + LicenseManager *enterpriseApi.LicenseManager +} + +// RunStandaloneDeploymentWorkflow deploys a standalone instance and verifies it's ready +func RunStandaloneDeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string) *WorkflowResult { + standalone, err := deployment.DeployStandalone(ctx, name, "", "") + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, name, standalone) + + return &WorkflowResult{Standalone: standalone} +} + +// RunC3DeploymentWorkflow deploys a C3 cluster (CM + IDXC + SHC) and verifies all components are ready +func RunC3DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, mcRef string) *WorkflowResult { + err := deployment.DeploySingleSiteCluster(ctx, name, indexerReplicas, true, mcRef) + Expect(err).To(Succeed(), "Unable to deploy C3 cluster") + + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + return &WorkflowResult{} +} + +// RunM4DeploymentWorkflow deploys a M4 multisite cluster and verifies all components are ready +func RunM4DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, siteCount int, mcRef string) *WorkflowResult { + err := deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) + Expect(err).To(Succeed(), "Unable to deploy M4 cluster") + + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + return &WorkflowResult{} +} + +// RunM1DeploymentWorkflow deploys a M1 multisite indexer cluster (no SHC) and verifies components +func RunM1DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, siteCount int) *WorkflowResult { + err := deployment.DeployMultisiteCluster(ctx, name, indexerReplicas, siteCount, "") + Expect(err).To(Succeed(), "Unable to deploy M1 cluster") + + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + return &WorkflowResult{} +} + +// RunStandaloneWithServiceAccountWorkflow deploys standalone with a service account +func RunStandaloneWithServiceAccountWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, serviceAccountName string) *WorkflowResult { + testcaseEnvInst.CreateServiceAccount(serviceAccountName) + + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + ServiceAccount: serviceAccountName, + }, + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, name, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone with service account") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, name, standalone) + + standalonePodName := fmt.Sprintf(StandalonePod, name, 0) + testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + + return &WorkflowResult{Standalone: standalone} +} + +// RunDeleteStandaloneWorkflow deploys and deletes a standalone instance +func RunDeleteStandaloneWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string) { + result := RunStandaloneDeploymentWorkflow(ctx, deployment, testcaseEnvInst, name) + + err := deployment.DeleteCR(ctx, result.Standalone) + Expect(err).To(Succeed(), "Unable to delete standalone instance") +} + +// RunDeleteC3Workflow deploys and deletes a C3 cluster +func RunDeleteC3Workflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int) { + RunC3DeploymentWorkflow(ctx, deployment, testcaseEnvInst, name, indexerReplicas, "") + + idxc := &enterpriseApi.IndexerCluster{} + idxcName := name + "-idxc" + err := deployment.GetInstance(ctx, idxcName, idxc) + Expect(err).To(Succeed(), "Unable to get Indexer Cluster instance") + + err = deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete Indexer Cluster") + + shc := &enterpriseApi.SearchHeadCluster{} + shcName := name + "-shc" + err = deployment.GetInstance(ctx, shcName, shc) + Expect(err).To(Succeed(), "Unable to get Search Head Cluster instance") + + err = deployment.DeleteCR(ctx, shc) + Expect(err).To(Succeed(), "Unable to delete Search Head Cluster") + + cm := &enterpriseApi.ClusterManager{} + err = deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Unable to get Cluster Manager instance") + + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Manager") +} diff --git a/test/testenv/testcaseenv.go b/test/testenv/testcaseenv.go index 4e0ccb3c4..a22981b67 100644 --- a/test/testenv/testcaseenv.go +++ b/test/testenv/testcaseenv.go @@ -118,11 +118,6 @@ func (testenv *TestCaseEnv) GetSplunkImage() string { return testenv.splunkImage } -// IsOperatorInstalledClusterWide returns if operator is installed clusterwide -func (testenv *TestCaseEnv) IsOperatorInstalledClusterWide() string { - return testenv.clusterWideOperator -} - func (testenv *TestCaseEnv) setup() error { testenv.Log.Info("testenv initializing.\n") diff --git a/test/testenv/util.go b/test/testenv/util.go index 90460849b..ae902f995 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -23,8 +23,6 @@ import ( "os" "os/exec" "path" - "reflect" - "sort" "strings" "time" @@ -1115,16 +1113,6 @@ func GetDirsOrFilesInPath(ctx context.Context, deployment *Deployment, podName s return strings.Fields(stdout), err } -// CompareStringSlices checks if two string slices are matching -func CompareStringSlices(stringOne []string, stringTwo []string) bool { - if len(stringOne) != len(stringTwo) { - return false - } - sort.Strings(stringOne) - sort.Strings(stringTwo) - return reflect.DeepEqual(stringOne, stringTwo) -} - // CheckStringInSlice check if string is present in a slice func CheckStringInSlice(stringSlice []string, compString string) bool { logf.Log.Info("Checking for string in slice", "String", compString, "String Slice", stringSlice) @@ -1170,16 +1158,6 @@ func GetPodUIDs(ns string) map[string]string { return splunkPodUIDs } -// DeletePod Delete pod in the namespace -func DeletePod(ns string, podName string) error { - _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() - if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) - return err - } - return nil -} - // DeleteOperatorPod Delete Operator Pod in the namespace func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { var podName string diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index 4b3bc1331..2393ffd88 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -316,15 +316,6 @@ func (testenv *TestCaseEnv) VerifyNoDisconnectedSHPresentOnCM(ctx context.Contex }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) } -// VerifyNoSHCInNamespace verify no SHC is present in namespace -func (testenv *TestCaseEnv) VerifyNoSHCInNamespace(deployment *Deployment) { - gomega.Eventually(func() bool { - shcStatus := SHCInNamespace(testenv.GetName()) - testenv.Log.Info("Verifying no Search Head Cluster is present in namespace", "Status", shcStatus) - return shcStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(false)) -} - // VerifyLicenseManagerReady verify LM is in ready status and does not flip flop func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deployment *Deployment) { LicenseManager := &enterpriseApi.LicenseManager{} @@ -439,15 +430,6 @@ func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deploymen }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) } -// VerifyRollingRestartFinished verify no rolling restart is active -func (testenv *TestCaseEnv) VerifyRollingRestartFinished(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() bool { - rollingRestartStatus := CheckRollingRestartStatus(ctx, deployment) - testenv.Log.Info("Rolling Restart Status", "active", rollingRestartStatus) - return rollingRestartStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(true)) -} - // VerifyConfOnPod Verify give conf and value on config file on pod func (testenv *TestCaseEnv) VerifyConfOnPod(deployment *Deployment, podName string, confFilePath string, config string, value string) { gomega.Consistently(func() bool {