feat: add siteopsSource parameter for cross-repo pipeline reuse

.github/actions/setup-siteops/action.yaml

@@ -10,6 +10,10 @@ inputs:
     description: "Include dev dependencies (pytest, pytest-cov)"
     required: false
     default: "false"
+  siteops-source:
+    description: "siteops install source (empty = local editable install)"
+    required: false
+    default: ""
 
 runs:
   using: "composite"
@@ -23,7 +27,7 @@ runs:
       uses: actions/cache@v5
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-siteops-${{ inputs.install-dev == 'true' && 'dev-' || '' }}${{ hashFiles('pyproject.toml') }}
+        key: ${{ runner.os }}-pip-siteops-${{ inputs.siteops-source || format('{0}{1}', inputs.install-dev == 'true' && 'dev-' || '', hashFiles('pyproject.toml')) }}
         restore-keys: |
           ${{ runner.os }}-pip-siteops-
           ${{ runner.os }}-pip-
@@ -32,9 +36,12 @@ runs:
       shell: bash
       env:
         INSTALL_DEV: ${{ inputs.install-dev }}
+        SITEOPS_SOURCE: ${{ inputs.siteops-source }}
       run: |
         pip install --upgrade pip
-        if [[ "$INSTALL_DEV" == "true" ]]; then
+        if [ -n "$SITEOPS_SOURCE" ]; then
+          pip install "$SITEOPS_SOURCE"
+        elif [[ "$INSTALL_DEV" == "true" ]]; then
           pip install -e ".[dev]"
         else
           pip install -e .

.github/workflows/_siteops-deploy.yaml

@@ -35,6 +35,11 @@ on:
         required: false
         type: boolean
         default: false
+      siteops-source:
+        description: "siteops install source (empty = local editable install)"
+        required: false
+        type: string
+        default: ""
     secrets:
       AZURE_CLIENT_ID:
         description: "Azure AD application client ID"
@@ -77,6 +82,8 @@ jobs:
 
       - name: Setup Site Ops
         uses: ./.github/actions/setup-siteops
+        with:
+          siteops-source: ${{ inputs.siteops-source }}
 
       - name: Validate inputs
         shell: bash
@@ -105,13 +112,13 @@ jobs:
         shell: bash
         run: |
           # Generate sites.local/ override files from SITE_OVERRIDES JSON secret.
-          # Supports nested paths using dot notation for parameters:
+          # Supports nested paths using dot notation at any depth:
           # {
           #   "munich-dev": {
           #     "subscription": "...",
           #     "resourceGroup": "...",
           #     "parameters.clusterName": "actual-cluster-name",
-          #     "parameters.customLocationName": "actual-custom-location"
+          #     "parameters.dataflowIdentity.clientId": "..."
           #   }
           # }
 
@@ -120,40 +127,38 @@ jobs:
             exit 0
           fi
 
-          if ! echo "$SITE_OVERRIDES" | jq empty 2>/dev/null; then
-            echo "::error::SITE_OVERRIDES secret is not valid JSON"
-            exit 1
-          fi
+          python3 - <<'GENERATE_OVERLAYS'
+          import json, os, re, sys, yaml
 
-          mkdir -p "$INPUT_WORKSPACE/sites.local"
-
-          for site_name in $(echo "$SITE_OVERRIDES" | jq -r 'keys[]'); do
-            # Sanitize site name - only allow alphanumeric, dash, underscore
-            if [[ ! "$site_name" =~ ^[a-zA-Z0-9_-]+$ ]]; then
-              echo "::error::Invalid site name in SITE_OVERRIDES: $site_name"
-              exit 1
-            fi
-
-            output_file="$INPUT_WORKSPACE/sites.local/${site_name}.yaml"
-
-            # Build the YAML file with proper nesting
-            # First, collect all entries and group by parent
-            {
-              # Write top-level keys first
-              echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \
-                '.[$site] | to_entries[] | select(.key | contains(".") | not) | "\(.key): \"\(.value)\""'
-
-              # Then write nested keys grouped by parent
-              for parent in $(echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \
-                '.[$site] | keys[] | select(contains(".")) | split(".")[0]' | sort -u); do
-                echo "${parent}:"
-                echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" --arg parent "$parent" \
-                  '.[$site] | to_entries[] | select(.key | startswith($parent + ".")) | "  \(.key | split(".")[1]): \"\(.value)\""'
-              done
-            } > "$output_file"
-
-            echo "✓ Generated override for: $site_name"
-          done
+          overrides = json.loads(os.environ["SITE_OVERRIDES"])
+          workspace = os.environ["INPUT_WORKSPACE"]
+          site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$")
+
+          sites_local = os.path.join(workspace, "sites.local")
+          os.makedirs(sites_local, exist_ok=True)
+
+          for site_name, values in overrides.items():
+              if not site_name_re.match(site_name):
+                  print(f"::error::Invalid site name in SITE_OVERRIDES: {site_name}")
+                  sys.exit(1)
+
+              # Expand dot-notation keys into nested dict
+              nested = {}
+              for key, value in values.items():
+                  parts = key.split(".")
+                  current = nested
+                  for part in parts[:-1]:
+                      current = current.setdefault(part, {})
+                  current[parts[-1]] = value
+
+              output_file = os.path.join(sites_local, f"{site_name}.yaml")
+              with open(output_file, "w") as f:
+                  yaml.safe_dump(nested, f, default_flow_style=False)
+
+              print(f"✓ Generated override for: {site_name}")
+
+          print(f"✓ Generated {len(overrides)} site override(s)")
+          GENERATE_OVERLAYS
 
           # Register each override value with GitHub's log masking so
           # subscription IDs, resource groups, etc. appear as *** in all
@@ -162,9 +167,6 @@ jobs:
             [[ -n "$val" ]] && echo "::add-mask::${val}"
           done
 
-          count=$(ls "$INPUT_WORKSPACE/sites.local/"*.yaml 2>/dev/null | wc -l)
-          echo "✓ Generated ${count} site override(s)"
-
       - name: Validate and show target sites
         shell: bash
         run: |

.pipelines/templates/setup-siteops.yaml

@@ -10,35 +10,49 @@ parameters:
     displayName: Include dev dependencies (pytest, pytest-cov)
     type: boolean
     default: false
+  - name: siteopsSource
+    displayName: siteops install source (empty = local editable install)
+    type: string
+    default: ''
+  - name: enableCache
+    displayName: Enable pip caching (disable for deployment jobs which lack caching scopes)
+    type: boolean
+    default: true
 
 steps:
   - task: UsePythonVersion@0
     displayName: Set up Python ${{ parameters.pythonVersion }}
     inputs:
       versionSpec: ${{ parameters.pythonVersion }}
 
-  - script: echo "##vso[task.setvariable variable=PIP_CACHE_DIR]$(pip cache dir)"
-    displayName: Resolve pip cache directory
+  - ${{ if eq(parameters.enableCache, true) }}:
+    - script: echo "##vso[task.setvariable variable=PIP_CACHE_DIR]$(pip cache dir)"
+      displayName: Resolve pip cache directory
 
-  - task: Cache@2
-    displayName: Cache pip packages
-    inputs:
-      key: pip | "$(Agent.OS)" | ${{ parameters.installDev }} | pyproject.toml
-      path: $(PIP_CACHE_DIR)
-      restoreKeys: |
-        pip | "$(Agent.OS)" | ${{ parameters.installDev }}
-        pip | "$(Agent.OS)"
+    - task: Cache@2
+      displayName: Cache pip packages
+      inputs:
+        ${{ if eq(parameters.siteopsSource, '') }}:
+          key: pip | "$(Agent.OS)" | ${{ parameters.installDev }} | pyproject.toml
+        ${{ else }}:
+          key: pip | "$(Agent.OS)" | "${{ parameters.siteopsSource }}"
+        path: $(PIP_CACHE_DIR)
+        restoreKeys: |
+          pip | "$(Agent.OS)"
 
   - script: |
       pip install --upgrade pip
-      if [ "$INSTALL_DEV" = "True" ]; then
+      if [ -n "$SITEOPS_SOURCE" ]; then
+        pip install "$SITEOPS_SOURCE"
+      elif [ "$INSTALL_DEV" = "True" ]; then
         pip install -e ".[dev]"
       else
         pip install -e .
       fi
     displayName: Install Site Ops
     env:
       INSTALL_DEV: ${{ parameters.installDev }}
+      SITEOPS_SOURCE: ${{ parameters.siteopsSource }}
 
   - script: siteops --version
     displayName: Verify Site Ops installation

.pipelines/templates/siteops-deploy.yaml

@@ -24,6 +24,10 @@ parameters:
     displayName: Preview only
     type: boolean
     default: false
+  - name: siteopsSource
+    displayName: siteops install source (empty = local editable install)
+    type: string
+    default: ''
 
 stages:
   - stage: deploy
@@ -42,6 +46,9 @@ stages:
                   persistCredentials: false
 
                 - template: setup-siteops.yaml
+                  parameters:
+                    siteopsSource: ${{ parameters.siteopsSource }}
+                    enableCache: false
 
                 - script: |
                     # Prevent path traversal attacks
@@ -69,13 +76,13 @@ stages:
 
                 - script: |
                     # Generate sites.local/ override files from SITE_OVERRIDES JSON secret.
-                    # Supports nested paths using dot notation for parameters:
+                    # Supports nested paths using dot notation at any depth:
                     # {
                     #   "munich-dev": {
                     #     "subscription": "...",
                     #     "resourceGroup": "...",
                     #     "parameters.clusterName": "actual-cluster-name",
-                    #     "parameters.customLocationName": "actual-custom-location"
+                    #     "parameters.dataflowIdentity.clientId": "..."
                     #   }
                     # }
 
@@ -84,49 +91,45 @@ stages:
                       exit 0
                     fi
 
-                    if ! echo "$SITE_OVERRIDES" | jq empty 2>/dev/null; then
-                      echo "##vso[task.logissue type=error]SITE_OVERRIDES secret is not valid JSON"
-                      exit 1
-                    fi
+                    python3 - <<'GENERATE_OVERLAYS'
+                    import json, os, re, sys, yaml
 
-                    mkdir -p "$WORKSPACE/sites.local"
+                    overrides = json.loads(os.environ["SITE_OVERRIDES"])
+                    workspace = os.environ["WORKSPACE"]
+                    site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$")
 
-                    for site_name in $(echo "$SITE_OVERRIDES" | jq -r 'keys[]'); do
-                      # Sanitize site name - only allow alphanumeric, dash, underscore
-                      if [[ ! "$site_name" =~ ^[a-zA-Z0-9_-]+$ ]]; then
-                        echo "##vso[task.logissue type=error]Invalid site name in SITE_OVERRIDES: $site_name"
-                        exit 1
-                      fi
+                    sites_local = os.path.join(workspace, "sites.local")
+                    os.makedirs(sites_local, exist_ok=True)
 
-                      output_file="$WORKSPACE/sites.local/${site_name}.yaml"
+                    for site_name, values in overrides.items():
+                        if not site_name_re.match(site_name):
+                            print(f"##vso[task.logissue type=error]Invalid site name in SITE_OVERRIDES: {site_name}")
+                            sys.exit(1)
 
-                      # Build the YAML file with proper nesting
-                      {
-                        # Write top-level keys first
-                        echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \
-                          '.[$site] | to_entries[] | select(.key | contains(".") | not) | "\(.key): \"\(.value)\""'
+                        # Expand dot-notation keys into nested dict
+                        nested = {}
+                        for key, value in values.items():
+                            parts = key.split(".")
+                            current = nested
+                            for part in parts[:-1]:
+                                current = current.setdefault(part, {})
+                            current[parts[-1]] = value
 
-                        # Then write nested keys grouped by parent
-                        for parent in $(echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \
-                          '.[$site] | keys[] | select(contains(".")) | split(".")[0]' | sort -u); do
-                          echo "${parent}:"
-                          echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" --arg parent "$parent" \
-                            '.[$site] | to_entries[] | select(.key | startswith($parent + ".")) | "  \(.key | split(".")[1]): \"\(.value)\""'
-                        done
-                      } > "$output_file"
+                        output_file = os.path.join(sites_local, f"{site_name}.yaml")
+                        with open(output_file, "w") as f:
+                            yaml.safe_dump(nested, f, default_flow_style=False)
 
-                      echo "✓ Generated override for: $site_name"
-                    done
+                        print(f"✓ Generated override for: {site_name}")
+
+                    print(f"✓ Generated {len(overrides)} site override(s)")
+                    GENERATE_OVERLAYS
 
                     # Mask each override value in ADO logs.
                     # ADO masking is additive — once a value is registered as secret,
                     # it stays masked for the remainder of the job.
                     echo "$SITE_OVERRIDES" | jq -r '.. | strings' | while IFS= read -r val; do
                       [[ -n "$val" ]] && echo "##vso[task.setvariable variable=masked;issecret=true]${val}"
                     done
-
-                    count=$(ls "$WORKSPACE/sites.local/"*.yaml 2>/dev/null | wc -l)
-                    echo "✓ Generated ${count} site override(s)"
                   displayName: Setup site overrides
                   env:
                     WORKSPACE: ${{ parameters.workspace }}