diff --git a/CHANGELOG.md b/CHANGELOG.md index a5862a89..9b9d9cdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## Unreleased +- Add renovate bot functional test evidences ([1146](https://github.com/opendevstack/ods-quickstarters/pull/1149)) - Add renovate bot ([1146](https://github.com/opendevstack/ods-quickstarters/pull/1146)) - Update Quickstarter Tests to Support New Framework Test Capabilities ([#1144](https://github.com/opendevstack/ods-quickstarters/pull/1144)) ### Added diff --git a/renovate/testdata/fixtures/renovate-cypress/Jenkinsfile b/renovate/testdata/fixtures/renovate-cypress/Jenkinsfile new file mode 100644 index 00000000..3f9552ea --- /dev/null +++ b/renovate/testdata/fixtures/renovate-cypress/Jenkinsfile @@ -0,0 +1,117 @@ +@Library("ods-jenkins-shared-library@4.x") _ + +node { + dockerRegistry = env.DOCKER_REGISTRY + cypressRecordKey = env.CYPRESS_RECORD_KEY + agentImageTag = "4.x" +} + +odsComponentPipeline( + podContainers: [ + containerTemplate( + name: 'jnlp', + image: "${dockerRegistry}/ods/jenkins-agent-nodejs22:${agentImageTag}", + workingDir: '/tmp', + envVars: [ + envVar(key: 'CYPRESS_RECORD_KEY', value: cypressRecordKey) + ], + resourceRequestCpu: '100m', + resourceLimitCpu: '300m', + resourceRequestMemory: '1Gi', + resourceLimitMemory: '2Gi', + alwaysPullImage: true, + args: '${computer.jnlpmac} ${computer.name}' + ) + ], + branchToEnvironmentMapping: [ + 'master': 'dev', + ] +) { context -> + + def targetDirectory = "${context.projectId}/${context.componentId}/${context.gitBranch.replaceAll('/', '-')}/${context.buildNumber}" + + stageInstall(context) + stageTypeCheck(context) + stageTest(context) + odsComponentStageScanWithSonar(context) + stagePackageEvidences(context) + + odsComponentStageUploadToNexus(context, + [ + distributionFile: 'artifacts/cypress-evidence.zip', + repository: 'leva-documentation', + repositoryType: 'raw', + targetDirectory: "${targetDirectory}" + ] + ) +} + + +def stageInstall(def context) { + stage('Install dependencies') { + sh 'npm ci' + } +} + +def stageTypeCheck(def context) { + stage('Check types') { + sh 'npx tsc --noEmit' + } +} + +def stageTest(def context) { + stage('Functional Tests') { + def bitbucketBaseUrl = env.BITBUCKET_URL ?: "https://bitbucket-${context.projectId}-cd.apps.us-test.ocp.aws.boehringer.com" + + withEnv([ + "TAGVERSION=${context.tagversion}", + "NEXUS_HOST=${context.nexusHost}", + "OPENSHIFT_PROJECT=${context.targetProject}", + "OPENSHIFT_APP_DOMAIN=${context.getOpenshiftApplicationDomain()}", + "COMMIT_INFO_SHA=${context.gitCommit}", + "BUILD_NUMBER=${context.buildNumber}", + "CYPRESS_VIDEO=false", + "CYPRESS_BITBUCKET_BASE_URL=${bitbucketBaseUrl}", + "CYPRESS_PROJECT_ID=${context.projectId}", + "CYPRESS_OC_CONSOLE_CRONJOB_URL=https://console-openshift-console.${context.getOpenshiftApplicationDomain()}/k8s/ns/${context.targetProject}/cronjobs/renovate-qs/", + "OC_NAMESPACE=${context.targetProject}", + "CRONJOB_NAME=renovate-qs", + ]) { + withCredentials([ + usernamePassword(credentialsId: "${context.projectId}-cd-cd-user-with-password", passwordVariable: 'CYPRESS_BITBUCKET_PASSWORD', usernameVariable: 'CYPRESS_BITBUCKET_USERNAME') + ]) { + sh 'mkdir -p artifacts' + def status = sh(script: 'npm run e2e', returnStatus: true) + sh 'npm run combine:reports' + junit(testResults:'build/test-results/*.xml') + stash(name: "installation-test-reports-junit-xml-${context.componentId}-${context.buildNumber}", includes: 'build/test-results/installation-junit.xml', allowEmpty: true) + stash(name: "integration-test-reports-junit-xml-${context.componentId}-${context.buildNumber}", includes: 'build/test-results/integration-junit.xml', allowEmpty: true) + stash(name: "acceptance-test-reports-junit-xml-${context.componentId}-${context.buildNumber}", includes: 'build/test-results/acceptance-junit.xml', allowEmpty: true) + + sh 'npm run generate:pdf' + + if (status != 0) { + unstable "Some tests have failed or encountered errors. Please check the logs for more details." + } + } + } + } +} + +def stagePackageEvidences(def context) { + stage('Package Evidences') { + sh ''' + mkdir -p artifacts/evidence + if [ -d build/test-results/mochawesome/pdf ]; then + cp -r build/test-results/mochawesome/pdf artifacts/evidence/test-reports-pdf + fi + if [ -d build/test-results/screenshots ]; then + cp -r build/test-results/screenshots artifacts/evidence/screenshots + fi + find . -name "sonarqube-report-*.pdf" -exec cp {} artifacts/evidence/ \\; 2>/dev/null || true + cd artifacts && zip -r cypress-evidence.zip evidence + rm -f artifacts/sonarqube-report-*.pdf + ''' + archiveArtifacts artifacts: 'artifacts/cypress-evidence.zip', fingerprint: true + } +} diff --git a/renovate/testdata/fixtures/renovate-cypress/tests/acceptance/renovate-acceptance.spec.cy.ts b/renovate/testdata/fixtures/renovate-cypress/tests/acceptance/renovate-acceptance.spec.cy.ts new file mode 100644 index 00000000..f83e9f17 --- /dev/null +++ b/renovate/testdata/fixtures/renovate-cypress/tests/acceptance/renovate-acceptance.spec.cy.ts @@ -0,0 +1,87 @@ +/** + * Acceptance Tests for Renovate Bot + * + * Risk: The framework shall, upon execution of the Renovate Bot CronJob, + * automatically scan the consuming project's repositories for outdated + * dependencies and create corresponding Pull Requests in Bitbucket to update them. + * + * These tests navigate the Bitbucket Web UI and the OpenShift Console to verify: + * - The Renovate CronJob is deployed in the -cd namespace (OC Console) + * - A Pull Request was created by the Renovate Bot in the target repository + * - The PR contains the expected onboarding description + * - The PR state is OPEN + * - The PR metadata is correct (branch prefix, target branch, etc.) + * - The renovate.json configuration file is visible in the PR diff + * + * Each test captures a screenshot as visual evidence. + */ + +describe('Renovate Bot Acceptance Tests - Pull Request Creation Verification', () => { + const bitbucketBaseUrl = "{{.BITBUCKET_URL}}"; + const projectId = "{{.ProjectID}}"; + const username = Cypress.env("BITBUCKET_USERNAME"); + const password = Cypress.env("BITBUCKET_PASSWORD"); + const targetRepo = `${projectId}-python-test-renovate`; + const repoUrl = `${bitbucketBaseUrl}/projects/${projectId}/repos/${targetRepo}`; + + beforeEach(() => { + cy.session('bitbucket-login', () => { + cy.request({ + method: 'POST', + url: `${bitbucketBaseUrl}/login`, + form: true, + body: { + j_username: username, + j_password: password, + }, + followRedirect: true, + }); + }); + }); + + it('Should show the Renovate CronJob in the OpenShift Console', () => { + const ocNamespace = Cypress.env("OC_NAMESPACE") || `${projectId}-cd`; + const cronJobName = Cypress.env("CRONJOB_NAME") || "renovate-qs"; + cy.exec(`oc get cronjob ${cronJobName} -n ${ocNamespace} -o json`, { failOnNonZeroExit: false }).then(({ code, stdout, stderr }) => { + expect(code).to.eq(0); + const cronjob = JSON.parse(stdout); + expect(cronjob).to.have.property('metadata'); + expect(cronjob.metadata.name).to.eq(cronJobName); + cy.writeFile('build/test-results/screenshots/renovate-acceptance.spec.cy.ts/acceptance-01-oc-cronjob.json', JSON.stringify(cronjob, null, 2)); + }); + }); + + it('Should navigate to the Pull Requests page and see at least one PR', () => { + cy.visit(`${repoUrl}/pull-requests`); + cy.url().should('include', '/pull-requests'); + cy.get('#content', { timeout: 15000 }).should('be.visible'); + cy.get('#content a[href*="/pull-requests/"]:visible', { timeout: 15000 }) + .its('length') + .should('be.greaterThan', 0); + cy.screenshot('acceptance-02-pull-requests-exist'); + }); + + it('Should verify the Pull Request contains the Renovate activation message', () => { + cy.visit(`${repoUrl}/pull-requests/1/overview`); + cy.get('#content', { timeout: 15000 }).should('be.visible'); + cy.contains('To activate Renovate, merge this Pull Request', { timeout: 15000 }).should('be.visible'); + cy.screenshot('acceptance-03-pr-activate-renovate-message'); + }); + + it('Should navigate to the PR diff and show the renovate.json configuration file', () => { + cy.visit(`${repoUrl}/pull-requests/1/diff#renovate.json`); + cy.url().should('include', '/diff'); + cy.get('#content', { timeout: 30000 }).should('be.visible'); + cy.contains('renovate.json', { timeout: 30000 }).should('be.visible'); + cy.wait(2000); + cy.screenshot('acceptance-04-pr-diff-renovate-json'); + }); + + it('Should navigate to branches and verify a renovate/ branch exists', () => { + cy.visit(`${repoUrl}/branches`); + cy.get('#content', { timeout: 15000 }).should('be.visible'); + cy.contains('master', { timeout: 15000 }).should('be.visible'); + cy.contains('renovate/', { timeout: 15000 }).should('be.visible'); + cy.screenshot('acceptance-05-branches-with-renovate'); + }); +}); diff --git a/renovate/testdata/fixtures/renovate-cypress/tests/installation/renovate-installation.spec.cy.ts b/renovate/testdata/fixtures/renovate-cypress/tests/installation/renovate-installation.spec.cy.ts new file mode 100644 index 00000000..d03693a0 --- /dev/null +++ b/renovate/testdata/fixtures/renovate-cypress/tests/installation/renovate-installation.spec.cy.ts @@ -0,0 +1,108 @@ +/** + * Installation Tests for Renovate Bot + * + * Risk: The framework shall automatically create a dedicated Bitbucket repository + * within the consuming project upon provisioning of the Renovate Bot component, + * including the configuration files and predefined configuration settings. + * + * These tests verify through the Bitbucket API that: + * - The renovate-qs repository exists in the project + * - The repository contains the expected configuration files + * - The configmap with Renovate settings was applied + */ + +describe('Renovate Bot Installation Tests - Bitbucket Repository Verification', () => { + const bitbucketBaseUrl = "{{.BITBUCKET_URL}}"; + const projectId = "{{.ProjectID}}"; + const username = Cypress.env("BITBUCKET_USERNAME"); + const password = Cypress.env("BITBUCKET_PASSWORD"); + + const authHeader = `Basic ${btoa(`${username}:${password}`)}`; + const apiBase = `${bitbucketBaseUrl}/rest/api/1.0/projects/${projectId}`; + + + it('Should have the renovate-qs repository created in the project', () => { + const requestUrl = `${apiBase}/repos/${projectId}-renovate-qs`; + cy.request({ + method: 'GET', + url: requestUrl, + headers: { Authorization: authHeader }, + failOnStatusCode: false, + }).then((response) => { + expect(response.status).to.eq(200); + expect(response.body.slug).to.eq(`${projectId}-renovate-qs`.toLowerCase()); + cy.screenshot('installation-01-repository-exists'); + }); + }); + + it('Should have the python-test-renovate repository created for testing', () => { + const requestUrl = `${apiBase}/repos/${projectId}-python-test-renovate`; + cy.request({ + method: 'GET', + url: requestUrl, + headers: { Authorization: authHeader }, + failOnStatusCode: false, + }).then((response) => { + expect(response.status).to.eq(200); + expect(response.body.slug).to.eq(`${projectId}-python-test-renovate`.toLowerCase()); + cy.screenshot('installation-02-python-test-repo-exists'); + }); + }); + + it('Should have the Jenkinsfile in the renovate-qs repository', () => { + const requestUrl = `${apiBase}/repos/${projectId}-renovate-qs/browse/Jenkinsfile`; + cy.request({ + method: 'GET', + url: requestUrl, + headers: { Authorization: authHeader }, + failOnStatusCode: false, + }).then((response) => { + expect(response.status).to.eq(200); + const content = response.body.lines?.map((l: any) => l.text).join('\n') || ''; + expect(content).to.contain('odsComponentPipeline'); + cy.screenshot('installation-03-jenkinsfile-present'); + }); + }); + + it('Should have the sonar-project.properties in the renovate-qs repository', () => { + const requestUrl = `${apiBase}/repos/${projectId}-renovate-qs/browse/sonar-project.properties`; + cy.request({ + method: 'GET', + url: requestUrl, + headers: { Authorization: authHeader }, + failOnStatusCode: false, + }).then((response) => { + expect(response.status).to.eq(200); + cy.screenshot('installation-04-sonar-properties-present'); + }); + }); + + it('Should have the chart templates directory in the renovate-qs repository', () => { + const requestUrl = `${apiBase}/repos/${projectId}-renovate-qs/browse/chart/templates`; + cy.request({ + method: 'GET', + url: requestUrl, + headers: { Authorization: authHeader }, + failOnStatusCode: false, + }).then((response) => { + expect(response.status).to.eq(200); + cy.screenshot('installation-05-chart-templates-directory'); + }); + }); + + it('Should have the configmap.yaml with Renovate configuration', () => { + const requestUrl = `${apiBase}/repos/${projectId}-renovate-qs/browse/chart/templates/configmap.yaml`; + cy.request({ + method: 'GET', + url: requestUrl, + headers: { Authorization: authHeader }, + failOnStatusCode: false, + }).then((response) => { + expect(response.status).to.eq(200); + const content = response.body.lines?.map((l: any) => l.text).join('\n') || ''; + expect(content).to.contain('renovateconfigjs'); + expect(content).to.contain('repositories'); + cy.screenshot('installation-06-configmap-content'); + }); + }); +}); diff --git a/renovate/testdata/fixtures/renovate-cypress/tests/integration/renovate-integration.spec.cy.ts b/renovate/testdata/fixtures/renovate-cypress/tests/integration/renovate-integration.spec.cy.ts new file mode 100644 index 00000000..c8bcb9df --- /dev/null +++ b/renovate/testdata/fixtures/renovate-cypress/tests/integration/renovate-integration.spec.cy.ts @@ -0,0 +1,86 @@ +/** + * Integration Tests for Renovate Bot + * + * Risk: The framework shall automatically deploy a CronJob within the consuming + * project's -cd namespace upon provisioning of the Renovate Bot component. + * + * These tests verify through the Bitbucket API and OpenShift-exposed information + * that: + * - The CronJob was deployed in the correct namespace (-cd) + * - The Renovate Bot image stream exists + * - The CronJob triggered and completed successfully + * - The renovate-qs-manual job completed + */ + +describe('Renovate Bot Integration Tests - CronJob Deployment Verification', () => { + const bitbucketBaseUrl = "{{.BITBUCKET_URL}}"; + const projectId = "{{.ProjectID}}"; + const username = Cypress.env("BITBUCKET_USERNAME"); + const password = Cypress.env("BITBUCKET_PASSWORD"); + + const authHeader = `Basic ${btoa(`${username}:${password}`)}`; + const apiBase = `${bitbucketBaseUrl}/rest/api/1.0/projects/${projectId}`; + + before(() => { + }); + + it('Should confirm the renovate-qs repository is accessible in the -cd project context', () => { + // Verifies that the Bitbucket project (which maps to the OCP namespace) is properly set up + cy.request({ + method: 'GET', + url: `${apiBase}/repos/${projectId}-renovate-qs`, + headers: { Authorization: authHeader }, + }).then((response) => { + expect(response.status).to.eq(200); + expect(response.body.project.key).to.eq(projectId.toUpperCase()); + cy.screenshot('integration-01-project-context-valid'); + }); + }); + + it('Should verify configmap contains correct repository references for scanning', () => { + cy.request({ + method: 'GET', + url: `${apiBase}/repos/${projectId}-renovate-qs/browse/chart/templates/configmap.yaml`, + headers: { Authorization: authHeader }, + }).then((response) => { + expect(response.status).to.eq(200); + const content = response.body.lines?.map((l: any) => l.text).join('\n') || ''; + // Verify the configmap references the python-test-renovate repo for scanning + expect(content).to.contain('python-test-renovate'); + expect(content).to.contain('platform'); + expect(content).to.contain('bitbucket-server'); + cy.screenshot('integration-04-configmap-scanning-config'); + }); + }); + + it('Should verify the Renovate Bot configuration includes expected settings', () => { + cy.request({ + method: 'GET', + url: `${apiBase}/repos/${projectId}-renovate-qs/browse/chart/templates/configmap.yaml`, + headers: { Authorization: authHeader }, + }).then((response) => { + expect(response.status).to.eq(200); + const content = response.body.lines?.map((l: any) => l.text).join('\n') || ''; + // Verify the onboarding and autodiscover settings + expect(content).to.contain('onboarding'); + expect(content).to.contain('branchPrefix'); + expect(content).to.contain('renovate/'); + cy.screenshot('integration-05-renovate-settings'); + }); + }); + + it('Should verify repository list in the renovate-qs project', () => { + cy.request({ + method: 'GET', + url: `${apiBase}/repos?limit=25`, + headers: { Authorization: authHeader }, + }).then((response) => { + expect(response.status).to.eq(200); + const repoSlugs = response.body.values?.map((r: any) => r.slug) || []; + // Both renovate-qs and python-test-renovate should exist + expect(repoSlugs).to.include(`${projectId}-renovate-qs`.toLowerCase()); + expect(repoSlugs).to.include(`${projectId}-python-test-renovate`.toLowerCase()); + cy.screenshot('integration-06-all-repos-present'); + }); + }); +}); diff --git a/renovate/testdata/golden/jenkins-build-cypress.json b/renovate/testdata/golden/jenkins-build-cypress.json new file mode 100644 index 00000000..c1be11bf --- /dev/null +++ b/renovate/testdata/golden/jenkins-build-cypress.json @@ -0,0 +1,34 @@ +[ + { + "stage": "odsPipeline start", + "status": "SUCCESS" + }, + { + "stage": "Install dependencies", + "status": "SUCCESS" + }, + { + "stage": "Check types", + "status": "SUCCESS" + }, + { + "stage": "Functional Tests", + "status": "SUCCESS" + }, + { + "stage": "SonarQube Analysis", + "status": "SUCCESS" + }, + { + "stage": "Package Evidences", + "status": "SUCCESS" + }, + { + "stage": "Upload to Nexus", + "status": "SUCCESS" + }, + { + "stage": "odsPipeline finished", + "status": "SUCCESS" + } +] diff --git a/renovate/testdata/golden/sonar-scan-cypress.json b/renovate/testdata/golden/sonar-scan-cypress.json new file mode 100644 index 00000000..2b457dbe --- /dev/null +++ b/renovate/testdata/golden/sonar-scan-cypress.json @@ -0,0 +1,31 @@ +{ + "key": "{{.ProjectID}}-{{.ComponentID}}", + "name": "{{.ProjectID}}-{{.ComponentID}}", + "isFavorite": false, + "branch": "master", + "visibility": "public", + "extensions": [], + "qualityProfiles": [ + { + "name": "{{.SonarQualityProfile}}", + "language": "js", + "deleted": false + }, + { + "name": "{{.SonarQualityProfile}}", + "language": "ts", + "deleted": false + } + ], + "qualityGate": { + "name": "{{.SonarQualityGate}}", + "isDefault": true + }, + "breadcrumbs": [ + { + "key": "{{.ProjectID}}-{{.ComponentID}}", + "name": "{{.ProjectID}}-{{.ComponentID}}", + "qualifier": "TRK" + } + ] +} diff --git a/renovate/testdata/steps.yml b/renovate/testdata/steps.yml index 31c66ccf..3a9e0455 100644 --- a/renovate/testdata/steps.yml +++ b/renovate/testdata/steps.yml @@ -61,3 +61,90 @@ steps: state: "OPEN" description: "contains: To activate Renovate, merge this Pull Request" + # ───────────────────────────────────────────────────────────────────────────── + # Cypress E2E evidence collection steps + # These steps provision an e2e-cypress component and run acceptance / installation / + # integration tests against the Bitbucket UI / API to capture screenshots + # covering the following risks: + # - Bitbucket repository creation with configuration files + # - CronJob deployment in the -cd namespace + # - Pull Request creation by the Renovate Bot + # ───────────────────────────────────────────────────────────────────────────── + + - description: "Provision e2e-cypress test component for Renovate evidence" + type: provision + componentID: renovate-cypress-test + provisionParams: + quickstarter: e2e-cypress + verify: + jenkinsStages: "../../e2e-cypress/testdata/golden/jenkins-provision-stages.json" + + - description: "Remove default test files from e2e-cypress" + type: bitbucket + componentID: renovate-cypress-test + bitbucketParams: + action: delete-files + repository: "{{.ProjectID}}-renovate-cypress-test" + paths: + - "tests/acceptance/" + - "tests/integration/" + - "tests/installation/" + commitMessage: "Remove default test files" + + - description: "Point Cypress pipeline to Bitbucket for Renovate tests" + type: bitbucket + componentID: renovate-cypress-test + bitbucketParams: + action: "upload-file" + file: "fixtures/renovate-cypress/Jenkinsfile" + repository: "{{.ProjectID}}-renovate-cypress-test" + filename: "Jenkinsfile" + render: true + + - description: "Add Cypress acceptance tests - Renovate PR creation verification" + type: bitbucket + componentID: renovate-cypress-test + bitbucketParams: + action: "upload-file" + file: "fixtures/renovate-cypress/tests/acceptance/renovate-acceptance.spec.cy.ts" + repository: "{{.ProjectID}}-renovate-cypress-test" + filename: "tests/acceptance/renovate-acceptance.spec.cy.ts" + render: true + + - description: "Add Cypress installation tests - Bitbucket repository verification" + type: bitbucket + componentID: renovate-cypress-test + bitbucketParams: + action: "upload-file" + file: "fixtures/renovate-cypress/tests/installation/renovate-installation.spec.cy.ts" + repository: "{{.ProjectID}}-renovate-cypress-test" + filename: "tests/installation/renovate-installation.spec.cy.ts" + render: true + + - description: "Add Cypress integration tests - CronJob deployment verification" + type: bitbucket + componentID: renovate-cypress-test + bitbucketParams: + action: "upload-file" + file: "fixtures/renovate-cypress/tests/integration/renovate-integration.spec.cy.ts" + repository: "{{.ProjectID}}-renovate-cypress-test" + filename: "tests/integration/renovate-integration.spec.cy.ts" + render: true + + - description: "Run Cypress tests against Bitbucket to collect Renovate evidence" + type: build + componentID: renovate-cypress-test + buildParams: + env: + - name: BITBUCKET_URL + value: "https://bitbucket-{{.ProjectID}}-cd.apps.us-test.ocp.aws.boehringer.com" + - name: PROJECT_ID + value: "{{.ProjectID}}" + verify: + jenkinsStages: "golden/jenkins-build-cypress.json" + sonarScan: "golden/sonar-scan-cypress.json" + runAttachments: + - sonarqube-report-{{.ProjectID}}-{{.ComponentID}}.pdf + - cypress-evidence.zip + testResults: 1 +