From c157b06ab00e8b5cb67e3fd4e568bfa01ede00cb Mon Sep 17 00:00:00 2001 From: Paymaun Heidari Date: Thu, 12 Mar 2026 18:04:47 -0700 Subject: [PATCH 1/3] Add siteopsSource parameter for cross-repo pipeline reuse Add siteopsSource/siteops-source parameter to setup templates on both ADO and GHA platforms. When provided, installs siteops from the given pip source instead of local editable install, enabling downstream repos to reference these templates cross-repo. Backward compatible: defaults to empty string, preserving existing pip install -e . behavior when no source is specified. Changes: - .github/actions/setup-siteops/action.yaml: siteops-source input - .github/workflows/_siteops-deploy.yaml: siteops-source input, threaded - .pipelines/templates/setup-siteops.yaml: siteopsSource parameter - .pipelines/templates/siteops-deploy.yaml: siteopsSource parameter, threaded --- .github/actions/setup-siteops/action.yaml | 9 ++++++++- .github/workflows/_siteops-deploy.yaml | 7 +++++++ .pipelines/templates/setup-siteops.yaml | 9 ++++++++- .pipelines/templates/siteops-deploy.yaml | 6 ++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-siteops/action.yaml b/.github/actions/setup-siteops/action.yaml index 44d5a73..49675d2 100644 --- a/.github/actions/setup-siteops/action.yaml +++ b/.github/actions/setup-siteops/action.yaml @@ -10,6 +10,10 @@ inputs: description: "Include dev dependencies (pytest, pytest-cov)" required: false default: "false" + siteops-source: + description: "siteops install source (empty = local editable install)" + required: false + default: "" runs: using: "composite" @@ -32,9 +36,12 @@ runs: shell: bash env: INSTALL_DEV: ${{ inputs.install-dev }} + SITEOPS_SOURCE: ${{ inputs.siteops-source }} run: | pip install --upgrade pip - if [[ "$INSTALL_DEV" == "true" ]]; then + if [ -n "$SITEOPS_SOURCE" ]; then + pip install "$SITEOPS_SOURCE" + elif [[ "$INSTALL_DEV" == "true" ]]; then pip install -e ".[dev]" else pip install -e . diff --git a/.github/workflows/_siteops-deploy.yaml b/.github/workflows/_siteops-deploy.yaml index 03d0a74..16c96b0 100644 --- a/.github/workflows/_siteops-deploy.yaml +++ b/.github/workflows/_siteops-deploy.yaml @@ -35,6 +35,11 @@ on: required: false type: boolean default: false + siteops-source: + description: "siteops install source (empty = local editable install)" + required: false + type: string + default: "" secrets: AZURE_CLIENT_ID: description: "Azure AD application client ID" @@ -77,6 +82,8 @@ jobs: - name: Setup Site Ops uses: ./.github/actions/setup-siteops + with: + siteops-source: ${{ inputs.siteops-source }} - name: Validate inputs shell: bash diff --git a/.pipelines/templates/setup-siteops.yaml b/.pipelines/templates/setup-siteops.yaml index ab49dce..9ae6f4e 100644 --- a/.pipelines/templates/setup-siteops.yaml +++ b/.pipelines/templates/setup-siteops.yaml @@ -10,6 +10,10 @@ parameters: displayName: Include dev dependencies (pytest, pytest-cov) type: boolean default: false + - name: siteopsSource + displayName: siteops install source (empty = local editable install) + type: string + default: '' steps: - task: UsePythonVersion@0 @@ -31,7 +35,9 @@ steps: - script: | pip install --upgrade pip - if [ "$INSTALL_DEV" = "True" ]; then + if [ -n "$SITEOPS_SOURCE" ]; then + pip install "$SITEOPS_SOURCE" + elif [ "$INSTALL_DEV" = "True" ]; then pip install -e ".[dev]" else pip install -e . @@ -39,6 +45,7 @@ steps: displayName: Install Site Ops env: INSTALL_DEV: ${{ parameters.installDev }} + SITEOPS_SOURCE: ${{ parameters.siteopsSource }} - script: siteops --version displayName: Verify Site Ops installation diff --git a/.pipelines/templates/siteops-deploy.yaml b/.pipelines/templates/siteops-deploy.yaml index 7664eb5..ecfdedd 100644 --- a/.pipelines/templates/siteops-deploy.yaml +++ b/.pipelines/templates/siteops-deploy.yaml @@ -24,6 +24,10 @@ parameters: displayName: Preview only type: boolean default: false + - name: siteopsSource + displayName: siteops install source (empty = local editable install) + type: string + default: '' stages: - stage: deploy @@ -42,6 +46,8 @@ stages: persistCredentials: false - template: setup-siteops.yaml + parameters: + siteopsSource: ${{ parameters.siteopsSource }} - script: | # Prevent path traversal attacks From 87438006f7c79d25389350a76cae0978526b10cb Mon Sep 17 00:00:00 2001 From: Paymaun Heidari Date: Fri, 13 Mar 2026 16:10:41 -0700 Subject: [PATCH 2/3] feat: support arbitrary nesting depth in SITE_OVERRIDES Replace shell/jq dot-notation parser with Python + PyYAML for generating sites.local overlay files. The previous implementation only handled one level of nesting (e.g. parameters.clusterName). The new implementation correctly expands any depth of dot-notation keys (e.g. parameters.dataflowIdentity.clientId) into properly nested YAML. Both GHA and ADO templates updated symmetrically. --- .github/workflows/_siteops-deploy.yaml | 71 +++++++++++------------- .pipelines/templates/siteops-deploy.yaml | 60 ++++++++++---------- 2 files changed, 61 insertions(+), 70 deletions(-) diff --git a/.github/workflows/_siteops-deploy.yaml b/.github/workflows/_siteops-deploy.yaml index 16c96b0..83f91a2 100644 --- a/.github/workflows/_siteops-deploy.yaml +++ b/.github/workflows/_siteops-deploy.yaml @@ -112,13 +112,13 @@ jobs: shell: bash run: | # Generate sites.local/ override files from SITE_OVERRIDES JSON secret. - # Supports nested paths using dot notation for parameters: + # Supports nested paths using dot notation at any depth: # { # "munich-dev": { # "subscription": "...", # "resourceGroup": "...", # "parameters.clusterName": "actual-cluster-name", - # "parameters.customLocationName": "actual-custom-location" + # "parameters.dataflowIdentity.clientId": "..." # } # } @@ -127,40 +127,38 @@ jobs: exit 0 fi - if ! echo "$SITE_OVERRIDES" | jq empty 2>/dev/null; then - echo "::error::SITE_OVERRIDES secret is not valid JSON" - exit 1 - fi + python3 - <<'GENERATE_OVERLAYS' + import json, os, re, sys, yaml - mkdir -p "$INPUT_WORKSPACE/sites.local" - - for site_name in $(echo "$SITE_OVERRIDES" | jq -r 'keys[]'); do - # Sanitize site name - only allow alphanumeric, dash, underscore - if [[ ! "$site_name" =~ ^[a-zA-Z0-9_-]+$ ]]; then - echo "::error::Invalid site name in SITE_OVERRIDES: $site_name" - exit 1 - fi - - output_file="$INPUT_WORKSPACE/sites.local/${site_name}.yaml" - - # Build the YAML file with proper nesting - # First, collect all entries and group by parent - { - # Write top-level keys first - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | to_entries[] | select(.key | contains(".") | not) | "\(.key): \"\(.value)\""' - - # Then write nested keys grouped by parent - for parent in $(echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | keys[] | select(contains(".")) | split(".")[0]' | sort -u); do - echo "${parent}:" - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" --arg parent "$parent" \ - '.[$site] | to_entries[] | select(.key | startswith($parent + ".")) | " \(.key | split(".")[1]): \"\(.value)\""' - done - } > "$output_file" - - echo "✓ Generated override for: $site_name" - done + overrides = json.loads(os.environ["SITE_OVERRIDES"]) + workspace = os.environ["INPUT_WORKSPACE"] + site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$") + + sites_local = os.path.join(workspace, "sites.local") + os.makedirs(sites_local, exist_ok=True) + + for site_name, values in overrides.items(): + if not site_name_re.match(site_name): + print(f"::error::Invalid site name in SITE_OVERRIDES: {site_name}") + sys.exit(1) + + # Expand dot-notation keys into nested dict + nested = {} + for key, value in values.items(): + parts = key.split(".") + current = nested + for part in parts[:-1]: + current = current.setdefault(part, {}) + current[parts[-1]] = value + + output_file = os.path.join(sites_local, f"{site_name}.yaml") + with open(output_file, "w") as f: + yaml.safe_dump(nested, f, default_flow_style=False) + + print(f"✓ Generated override for: {site_name}") + + print(f"✓ Generated {len(overrides)} site override(s)") + GENERATE_OVERLAYS # Register each override value with GitHub's log masking so # subscription IDs, resource groups, etc. appear as *** in all @@ -169,9 +167,6 @@ jobs: [[ -n "$val" ]] && echo "::add-mask::${val}" done - count=$(ls "$INPUT_WORKSPACE/sites.local/"*.yaml 2>/dev/null | wc -l) - echo "✓ Generated ${count} site override(s)" - - name: Validate and show target sites shell: bash run: | diff --git a/.pipelines/templates/siteops-deploy.yaml b/.pipelines/templates/siteops-deploy.yaml index ecfdedd..0e7e838 100644 --- a/.pipelines/templates/siteops-deploy.yaml +++ b/.pipelines/templates/siteops-deploy.yaml @@ -75,13 +75,13 @@ stages: - script: | # Generate sites.local/ override files from SITE_OVERRIDES JSON secret. - # Supports nested paths using dot notation for parameters: + # Supports nested paths using dot notation at any depth: # { # "munich-dev": { # "subscription": "...", # "resourceGroup": "...", # "parameters.clusterName": "actual-cluster-name", - # "parameters.customLocationName": "actual-custom-location" + # "parameters.dataflowIdentity.clientId": "..." # } # } @@ -90,39 +90,38 @@ stages: exit 0 fi - if ! echo "$SITE_OVERRIDES" | jq empty 2>/dev/null; then - echo "##vso[task.logissue type=error]SITE_OVERRIDES secret is not valid JSON" - exit 1 - fi + python3 - <<'GENERATE_OVERLAYS' + import json, os, re, sys, yaml - mkdir -p "$WORKSPACE/sites.local" + overrides = json.loads(os.environ["SITE_OVERRIDES"]) + workspace = os.environ["WORKSPACE"] + site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$") - for site_name in $(echo "$SITE_OVERRIDES" | jq -r 'keys[]'); do - # Sanitize site name - only allow alphanumeric, dash, underscore - if [[ ! "$site_name" =~ ^[a-zA-Z0-9_-]+$ ]]; then - echo "##vso[task.logissue type=error]Invalid site name in SITE_OVERRIDES: $site_name" - exit 1 - fi + sites_local = os.path.join(workspace, "sites.local") + os.makedirs(sites_local, exist_ok=True) - output_file="$WORKSPACE/sites.local/${site_name}.yaml" + for site_name, values in overrides.items(): + if not site_name_re.match(site_name): + print(f"##vso[task.logissue type=error]Invalid site name in SITE_OVERRIDES: {site_name}") + sys.exit(1) - # Build the YAML file with proper nesting - { - # Write top-level keys first - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | to_entries[] | select(.key | contains(".") | not) | "\(.key): \"\(.value)\""' + # Expand dot-notation keys into nested dict + nested = {} + for key, value in values.items(): + parts = key.split(".") + current = nested + for part in parts[:-1]: + current = current.setdefault(part, {}) + current[parts[-1]] = value - # Then write nested keys grouped by parent - for parent in $(echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | keys[] | select(contains(".")) | split(".")[0]' | sort -u); do - echo "${parent}:" - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" --arg parent "$parent" \ - '.[$site] | to_entries[] | select(.key | startswith($parent + ".")) | " \(.key | split(".")[1]): \"\(.value)\""' - done - } > "$output_file" + output_file = os.path.join(sites_local, f"{site_name}.yaml") + with open(output_file, "w") as f: + yaml.safe_dump(nested, f, default_flow_style=False) - echo "✓ Generated override for: $site_name" - done + print(f"✓ Generated override for: {site_name}") + + print(f"✓ Generated {len(overrides)} site override(s)") + GENERATE_OVERLAYS # Mask each override value in ADO logs. # ADO masking is additive — once a value is registered as secret, @@ -130,9 +129,6 @@ stages: echo "$SITE_OVERRIDES" | jq -r '.. | strings' | while IFS= read -r val; do [[ -n "$val" ]] && echo "##vso[task.setvariable variable=masked;issecret=true]${val}" done - - count=$(ls "$WORKSPACE/sites.local/"*.yaml 2>/dev/null | wc -l) - echo "✓ Generated ${count} site override(s)" displayName: Setup site overrides env: WORKSPACE: ${{ parameters.workspace }} From df03a1948f5c1bd93a68191b7a5c582091ba25a5 Mon Sep 17 00:00:00 2001 From: Paymaun Heidari Date: Fri, 13 Mar 2026 16:41:48 -0700 Subject: [PATCH 3/3] fix: improve cross-repo caching for ADO and GHA templates GHA: Use siteops-source as cache key when provided, falling back to hashFiles(pyproject.toml) for local installs. Prevents weak/colliding cache keys when cross-repo consumers lack pyproject.toml. ADO: Same conditional cache key logic using siteopsSource parameter. ADO: Add enableCache parameter (default true) to setup-siteops template. The deploy template passes enableCache=false since ADO deployment jobs cannot access pipeline caching scopes. This eliminates the post-job Cache error entirely rather than masking it with continueOnError. --- .github/actions/setup-siteops/action.yaml | 2 +- .pipelines/templates/setup-siteops.yaml | 27 ++++++++++++++--------- .pipelines/templates/siteops-deploy.yaml | 1 + 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup-siteops/action.yaml b/.github/actions/setup-siteops/action.yaml index 49675d2..78a6e18 100644 --- a/.github/actions/setup-siteops/action.yaml +++ b/.github/actions/setup-siteops/action.yaml @@ -27,7 +27,7 @@ runs: uses: actions/cache@v5 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-siteops-${{ inputs.install-dev == 'true' && 'dev-' || '' }}${{ hashFiles('pyproject.toml') }} + key: ${{ runner.os }}-pip-siteops-${{ inputs.siteops-source || format('{0}{1}', inputs.install-dev == 'true' && 'dev-' || '', hashFiles('pyproject.toml')) }} restore-keys: | ${{ runner.os }}-pip-siteops- ${{ runner.os }}-pip- diff --git a/.pipelines/templates/setup-siteops.yaml b/.pipelines/templates/setup-siteops.yaml index 9ae6f4e..9b6175f 100644 --- a/.pipelines/templates/setup-siteops.yaml +++ b/.pipelines/templates/setup-siteops.yaml @@ -14,6 +14,10 @@ parameters: displayName: siteops install source (empty = local editable install) type: string default: '' + - name: enableCache + displayName: Enable pip caching (disable for deployment jobs which lack caching scopes) + type: boolean + default: true steps: - task: UsePythonVersion@0 @@ -21,17 +25,20 @@ steps: inputs: versionSpec: ${{ parameters.pythonVersion }} - - script: echo "##vso[task.setvariable variable=PIP_CACHE_DIR]$(pip cache dir)" - displayName: Resolve pip cache directory + - ${{ if eq(parameters.enableCache, true) }}: + - script: echo "##vso[task.setvariable variable=PIP_CACHE_DIR]$(pip cache dir)" + displayName: Resolve pip cache directory - - task: Cache@2 - displayName: Cache pip packages - inputs: - key: pip | "$(Agent.OS)" | ${{ parameters.installDev }} | pyproject.toml - path: $(PIP_CACHE_DIR) - restoreKeys: | - pip | "$(Agent.OS)" | ${{ parameters.installDev }} - pip | "$(Agent.OS)" + - task: Cache@2 + displayName: Cache pip packages + inputs: + ${{ if eq(parameters.siteopsSource, '') }}: + key: pip | "$(Agent.OS)" | ${{ parameters.installDev }} | pyproject.toml + ${{ else }}: + key: pip | "$(Agent.OS)" | "${{ parameters.siteopsSource }}" + path: $(PIP_CACHE_DIR) + restoreKeys: | + pip | "$(Agent.OS)" - script: | pip install --upgrade pip diff --git a/.pipelines/templates/siteops-deploy.yaml b/.pipelines/templates/siteops-deploy.yaml index 0e7e838..c417bee 100644 --- a/.pipelines/templates/siteops-deploy.yaml +++ b/.pipelines/templates/siteops-deploy.yaml @@ -48,6 +48,7 @@ stages: - template: setup-siteops.yaml parameters: siteopsSource: ${{ parameters.siteopsSource }} + enableCache: false - script: | # Prevent path traversal attacks