test: add test harness, workspace validation, and integration tests

.github/workflows/_siteops-deploy.yaml

@@ -111,58 +111,14 @@ jobs:
           SITE_OVERRIDES: ${{ secrets.SITE_OVERRIDES }}
         shell: bash
         run: |
-          # Generate sites.local/ override files from SITE_OVERRIDES JSON secret.
-          # Supports nested paths using dot notation at any depth:
-          # {
-          #   "munich-dev": {
-          #     "subscription": "...",
-          #     "resourceGroup": "...",
-          #     "parameters.clusterName": "actual-cluster-name",
-          #     "parameters.dataflowIdentity.clientId": "..."
-          #   }
-          # }
-
           if [[ -z "$SITE_OVERRIDES" ]]; then
             echo "No site overrides configured, using committed sites"
             exit 0
           fi
 
-          python3 - <<'GENERATE_OVERLAYS'
-          import json, os, re, sys, yaml
-
-          overrides = json.loads(os.environ["SITE_OVERRIDES"])
-          workspace = os.environ["INPUT_WORKSPACE"]
-          site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$")
-
-          sites_local = os.path.join(workspace, "sites.local")
-          os.makedirs(sites_local, exist_ok=True)
-
-          for site_name, values in overrides.items():
-              if not site_name_re.match(site_name):
-                  print(f"::error::Invalid site name in SITE_OVERRIDES: {site_name}")
-                  sys.exit(1)
-
-              # Expand dot-notation keys into nested dict
-              nested = {}
-              for key, value in values.items():
-                  parts = key.split(".")
-                  current = nested
-                  for part in parts[:-1]:
-                      current = current.setdefault(part, {})
-                  current[parts[-1]] = value
-
-              output_file = os.path.join(sites_local, f"{site_name}.yaml")
-              with open(output_file, "w") as f:
-                  yaml.safe_dump(nested, f, default_flow_style=False)
-
-              print(f"✓ Generated override for: {site_name}")
-
-          print(f"✓ Generated {len(overrides)} site override(s)")
-          GENERATE_OVERLAYS
+          echo "$SITE_OVERRIDES" | python3 scripts/generate-site-overrides.py "$INPUT_WORKSPACE"
 
-          # Register each override value with GitHub's log masking so
-          # subscription IDs, resource groups, etc. appear as *** in all
-          # subsequent step output (logs, summaries, artifacts).
+          # Mask override values in GitHub Actions logs
           echo "$SITE_OVERRIDES" | jq -r '.. | strings' | while IFS= read -r val; do
             [[ -n "$val" ]] && echo "::add-mask::${val}"
           done

.github/workflows/integration-test.yaml

@@ -0,0 +1,103 @@
+# Integration tests: deploy manifests against real Azure and assert outputs
+# On-demand only via workflow_dispatch
+
+name: Integration Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      manifest:
+        description: "Manifest to test"
+        type: choice
+        options:
+          - all
+          - aio-install
+          - secretsync
+          - opc-ua-solution
+        default: all
+      skip-cleanup:
+        description: "Skip resource cleanup (for debugging)"
+        type: boolean
+        default: false
+      selector:
+        description: "Site selector override (default: manifest selector)"
+        type: string
+        default: ""
+      environment:
+        description: "Target environment"
+        type: string
+        default: "dev"
+
+env:
+  PYTHONUNBUFFERED: "1"
+
+jobs:
+  integration-test:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Setup Site Ops
+        uses: ./.github/actions/setup-siteops
+        with:
+          install-dev: true
+
+      - name: Azure Login (OIDC)
+        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Select test scope
+        id: scope
+        run: |
+          MANIFEST="${{ inputs.manifest }}"
+          if [[ "$MANIFEST" == "all" ]]; then
+            echo "pytest_args=tests/integration/" >> $GITHUB_OUTPUT
+          else
+            echo "pytest_args=tests/integration/test_${MANIFEST//-/_}_manifest.py" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Mask secret values
+        env:
+          SITE_OVERRIDES: ${{ secrets.SITE_OVERRIDES }}
+        run: |
+          if [[ -n "$SITE_OVERRIDES" ]]; then
+            echo "$SITE_OVERRIDES" | jq -r '.. | strings' | while IFS= read -r val; do
+              [[ -n "$val" ]] && echo "::add-mask::${val}"
+            done
+          fi
+
+      - name: Run integration tests
+        env:
+          SITE_OVERRIDES: ${{ secrets.SITE_OVERRIDES }}
+          INTEGRATION_SKIP_CLEANUP: ${{ inputs.skip-cleanup }}
+          INTEGRATION_SELECTOR: ${{ inputs.selector }}
+        run: >
+          pytest ${{ steps.scope.outputs.pytest_args }}
+          -v --tb=long
+          -m integration
+          --junitxml=integration-results.xml
+
+      - name: Publish test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-test-results
+          path: integration-results.xml
+
+      - name: Azure Logout
+        if: always()
+        run: az logout
+        continue-on-error: true

.pipelines/integration-test.yaml

@@ -0,0 +1,115 @@
+# Integration tests: deploy manifests against real Azure and assert outputs
+# On-demand only via manual trigger
+# Equivalent to .github/workflows/integration-test.yaml
+
+trigger: none
+
+pr: none
+
+parameters:
+  - name: manifest
+    displayName: "Manifest to test"
+    type: string
+    default: all
+    values:
+      - all
+      - aio-install
+      - secretsync
+      - opc-ua-solution
+
+  - name: skipCleanup
+    displayName: "Skip resource cleanup (for debugging)"
+    type: boolean
+    default: false
+
+  - name: environment
+    displayName: Target environment
+    type: string
+    default: dev
+
+  - name: selector
+    displayName: "Site selector override (default: manifest selector)"
+    type: string
+    default: ""
+
+  - name: serviceConnections
+    displayName: Service connection per environment
+    type: object
+    default:
+      dev: azure-siteops
+      staging: azure-siteops
+      prod: azure-siteops
+
+  - name: secretGroups
+    displayName: Variable group per environment
+    type: object
+    default:
+      dev: siteops-secrets
+      staging: siteops-secrets
+      prod: siteops-secrets
+
+variables:
+  - group: ${{ parameters.secretGroups[parameters.environment] }}
+  - name: PYTHONUNBUFFERED
+    value: '1'
+
+pool:
+  vmImage: ubuntu-latest
+
+jobs:
+  - deployment: integration_test
+    displayName: Integration Tests
+    environment: ${{ parameters.environment }}
+
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+            - checkout: self
+              fetchDepth: 1
+              persistCredentials: false
+
+            - template: templates/setup-siteops.yaml
+              parameters:
+                installDev: true
+
+            - script: |
+                if [[ -n "$SITE_OVERRIDES" ]]; then
+                  echo "$SITE_OVERRIDES" | jq -r '.. | strings' | while IFS= read -r val; do
+                    [[ -n "$val" ]] && echo "##vso[task.setvariable variable=masked;issecret=true]${val}"
+                  done
+                fi
+              displayName: Mask secret values
+              env:
+                SITE_OVERRIDES: $(SITE_OVERRIDES)
+
+            - task: AzureCLI@2
+              displayName: Run integration tests
+              inputs:
+                azureSubscription: ${{ parameters.serviceConnections[parameters.environment] }}
+                scriptType: bash
+                scriptLocation: inlineScript
+                inlineScript: |
+                  MANIFEST="${{ parameters.manifest }}"
+                  if [[ "$MANIFEST" == "all" ]]; then
+                    PYTEST_ARGS="tests/integration/"
+                  else
+                    PYTEST_ARGS="tests/integration/test_${MANIFEST//-/_}_manifest.py"
+                  fi
+
+                  pytest $PYTEST_ARGS \
+                    -v --tb=long \
+                    -m integration \
+                    --junitxml=$(Build.ArtifactStagingDirectory)/integration-results.xml
+              env:
+                SITE_OVERRIDES: $(SITE_OVERRIDES)
+                INTEGRATION_SKIP_CLEANUP: ${{ parameters.skipCleanup }}
+                INTEGRATION_SELECTOR: ${{ parameters.selector }}
+
+            - task: PublishTestResults@2
+              displayName: Publish test results
+              condition: succeededOrFailed()
+              inputs:
+                testResultsFormat: JUnit
+                testResultsFiles: "$(Build.ArtifactStagingDirectory)/integration-results.xml"
+                testRunTitle: Integration Tests

.pipelines/templates/siteops-deploy.yaml

@@ -75,58 +75,14 @@ stages:
                     SELECTOR: ${{ parameters.selector }}
 
                 - script: |
-                    # Generate sites.local/ override files from SITE_OVERRIDES JSON secret.
-                    # Supports nested paths using dot notation at any depth:
-                    # {
-                    #   "munich-dev": {
-                    #     "subscription": "...",
-                    #     "resourceGroup": "...",
-                    #     "parameters.clusterName": "actual-cluster-name",
-                    #     "parameters.dataflowIdentity.clientId": "..."
-                    #   }
-                    # }
-
                     if [[ -z "$SITE_OVERRIDES" ]]; then
                       echo "No site overrides configured, using committed sites"
                       exit 0
                     fi
 
-                    python3 - <<'GENERATE_OVERLAYS'
-                    import json, os, re, sys, yaml
-
-                    overrides = json.loads(os.environ["SITE_OVERRIDES"])
-                    workspace = os.environ["WORKSPACE"]
-                    site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$")
-
-                    sites_local = os.path.join(workspace, "sites.local")
-                    os.makedirs(sites_local, exist_ok=True)
-
-                    for site_name, values in overrides.items():
-                        if not site_name_re.match(site_name):
-                            print(f"##vso[task.logissue type=error]Invalid site name in SITE_OVERRIDES: {site_name}")
-                            sys.exit(1)
-
-                        # Expand dot-notation keys into nested dict
-                        nested = {}
-                        for key, value in values.items():
-                            parts = key.split(".")
-                            current = nested
-                            for part in parts[:-1]:
-                                current = current.setdefault(part, {})
-                            current[parts[-1]] = value
-
-                        output_file = os.path.join(sites_local, f"{site_name}.yaml")
-                        with open(output_file, "w") as f:
-                            yaml.safe_dump(nested, f, default_flow_style=False)
-
-                        print(f"✓ Generated override for: {site_name}")
-
-                    print(f"✓ Generated {len(overrides)} site override(s)")
-                    GENERATE_OVERLAYS
+                    echo "$SITE_OVERRIDES" | python3 scripts/generate-site-overrides.py "$WORKSPACE"
 
-                    # Mask each override value in ADO logs.
-                    # ADO masking is additive — once a value is registered as secret,
-                    # it stays masked for the remainder of the job.
+                    # Mask override values in ADO logs
                     echo "$SITE_OVERRIDES" | jq -r '.. | strings' | while IFS= read -r val; do
                       [[ -n "$val" ]] && echo "##vso[task.setvariable variable=masked;issecret=true]${val}"
                     done

pyproject.toml

@@ -51,6 +51,9 @@ testpaths = ["tests"]
 python_files = ["test_*.py"]
 python_functions = ["test_*"]
 addopts = "-v --tb=short"
+markers = [
+    "integration: requires Azure credentials and real resources (deselect with '-m not integration')",
+]
 filterwarnings = [
     "ignore::DeprecationWarning",
 ]