diff --git a/.github/actions/setup-siteops/action.yaml b/.github/actions/setup-siteops/action.yaml index 44d5a73..78a6e18 100644 --- a/.github/actions/setup-siteops/action.yaml +++ b/.github/actions/setup-siteops/action.yaml @@ -10,6 +10,10 @@ inputs: description: "Include dev dependencies (pytest, pytest-cov)" required: false default: "false" + siteops-source: + description: "siteops install source (empty = local editable install)" + required: false + default: "" runs: using: "composite" @@ -23,7 +27,7 @@ runs: uses: actions/cache@v5 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-siteops-${{ inputs.install-dev == 'true' && 'dev-' || '' }}${{ hashFiles('pyproject.toml') }} + key: ${{ runner.os }}-pip-siteops-${{ inputs.siteops-source || format('{0}{1}', inputs.install-dev == 'true' && 'dev-' || '', hashFiles('pyproject.toml')) }} restore-keys: | ${{ runner.os }}-pip-siteops- ${{ runner.os }}-pip- @@ -32,9 +36,12 @@ runs: shell: bash env: INSTALL_DEV: ${{ inputs.install-dev }} + SITEOPS_SOURCE: ${{ inputs.siteops-source }} run: | pip install --upgrade pip - if [[ "$INSTALL_DEV" == "true" ]]; then + if [ -n "$SITEOPS_SOURCE" ]; then + pip install "$SITEOPS_SOURCE" + elif [[ "$INSTALL_DEV" == "true" ]]; then pip install -e ".[dev]" else pip install -e . diff --git a/.github/workflows/_siteops-deploy.yaml b/.github/workflows/_siteops-deploy.yaml index 03d0a74..83f91a2 100644 --- a/.github/workflows/_siteops-deploy.yaml +++ b/.github/workflows/_siteops-deploy.yaml @@ -35,6 +35,11 @@ on: required: false type: boolean default: false + siteops-source: + description: "siteops install source (empty = local editable install)" + required: false + type: string + default: "" secrets: AZURE_CLIENT_ID: description: "Azure AD application client ID" @@ -77,6 +82,8 @@ jobs: - name: Setup Site Ops uses: ./.github/actions/setup-siteops + with: + siteops-source: ${{ inputs.siteops-source }} - name: Validate inputs shell: bash @@ -105,13 +112,13 @@ jobs: shell: bash run: | # Generate sites.local/ override files from SITE_OVERRIDES JSON secret. - # Supports nested paths using dot notation for parameters: + # Supports nested paths using dot notation at any depth: # { # "munich-dev": { # "subscription": "...", # "resourceGroup": "...", # "parameters.clusterName": "actual-cluster-name", - # "parameters.customLocationName": "actual-custom-location" + # "parameters.dataflowIdentity.clientId": "..." # } # } @@ -120,40 +127,38 @@ jobs: exit 0 fi - if ! echo "$SITE_OVERRIDES" | jq empty 2>/dev/null; then - echo "::error::SITE_OVERRIDES secret is not valid JSON" - exit 1 - fi + python3 - <<'GENERATE_OVERLAYS' + import json, os, re, sys, yaml - mkdir -p "$INPUT_WORKSPACE/sites.local" - - for site_name in $(echo "$SITE_OVERRIDES" | jq -r 'keys[]'); do - # Sanitize site name - only allow alphanumeric, dash, underscore - if [[ ! "$site_name" =~ ^[a-zA-Z0-9_-]+$ ]]; then - echo "::error::Invalid site name in SITE_OVERRIDES: $site_name" - exit 1 - fi - - output_file="$INPUT_WORKSPACE/sites.local/${site_name}.yaml" - - # Build the YAML file with proper nesting - # First, collect all entries and group by parent - { - # Write top-level keys first - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | to_entries[] | select(.key | contains(".") | not) | "\(.key): \"\(.value)\""' - - # Then write nested keys grouped by parent - for parent in $(echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | keys[] | select(contains(".")) | split(".")[0]' | sort -u); do - echo "${parent}:" - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" --arg parent "$parent" \ - '.[$site] | to_entries[] | select(.key | startswith($parent + ".")) | " \(.key | split(".")[1]): \"\(.value)\""' - done - } > "$output_file" - - echo "✓ Generated override for: $site_name" - done + overrides = json.loads(os.environ["SITE_OVERRIDES"]) + workspace = os.environ["INPUT_WORKSPACE"] + site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$") + + sites_local = os.path.join(workspace, "sites.local") + os.makedirs(sites_local, exist_ok=True) + + for site_name, values in overrides.items(): + if not site_name_re.match(site_name): + print(f"::error::Invalid site name in SITE_OVERRIDES: {site_name}") + sys.exit(1) + + # Expand dot-notation keys into nested dict + nested = {} + for key, value in values.items(): + parts = key.split(".") + current = nested + for part in parts[:-1]: + current = current.setdefault(part, {}) + current[parts[-1]] = value + + output_file = os.path.join(sites_local, f"{site_name}.yaml") + with open(output_file, "w") as f: + yaml.safe_dump(nested, f, default_flow_style=False) + + print(f"✓ Generated override for: {site_name}") + + print(f"✓ Generated {len(overrides)} site override(s)") + GENERATE_OVERLAYS # Register each override value with GitHub's log masking so # subscription IDs, resource groups, etc. appear as *** in all @@ -162,9 +167,6 @@ jobs: [[ -n "$val" ]] && echo "::add-mask::${val}" done - count=$(ls "$INPUT_WORKSPACE/sites.local/"*.yaml 2>/dev/null | wc -l) - echo "✓ Generated ${count} site override(s)" - - name: Validate and show target sites shell: bash run: | diff --git a/.pipelines/templates/setup-siteops.yaml b/.pipelines/templates/setup-siteops.yaml index ab49dce..9b6175f 100644 --- a/.pipelines/templates/setup-siteops.yaml +++ b/.pipelines/templates/setup-siteops.yaml @@ -10,6 +10,14 @@ parameters: displayName: Include dev dependencies (pytest, pytest-cov) type: boolean default: false + - name: siteopsSource + displayName: siteops install source (empty = local editable install) + type: string + default: '' + - name: enableCache + displayName: Enable pip caching (disable for deployment jobs which lack caching scopes) + type: boolean + default: true steps: - task: UsePythonVersion@0 @@ -17,21 +25,26 @@ steps: inputs: versionSpec: ${{ parameters.pythonVersion }} - - script: echo "##vso[task.setvariable variable=PIP_CACHE_DIR]$(pip cache dir)" - displayName: Resolve pip cache directory + - ${{ if eq(parameters.enableCache, true) }}: + - script: echo "##vso[task.setvariable variable=PIP_CACHE_DIR]$(pip cache dir)" + displayName: Resolve pip cache directory - - task: Cache@2 - displayName: Cache pip packages - inputs: - key: pip | "$(Agent.OS)" | ${{ parameters.installDev }} | pyproject.toml - path: $(PIP_CACHE_DIR) - restoreKeys: | - pip | "$(Agent.OS)" | ${{ parameters.installDev }} - pip | "$(Agent.OS)" + - task: Cache@2 + displayName: Cache pip packages + inputs: + ${{ if eq(parameters.siteopsSource, '') }}: + key: pip | "$(Agent.OS)" | ${{ parameters.installDev }} | pyproject.toml + ${{ else }}: + key: pip | "$(Agent.OS)" | "${{ parameters.siteopsSource }}" + path: $(PIP_CACHE_DIR) + restoreKeys: | + pip | "$(Agent.OS)" - script: | pip install --upgrade pip - if [ "$INSTALL_DEV" = "True" ]; then + if [ -n "$SITEOPS_SOURCE" ]; then + pip install "$SITEOPS_SOURCE" + elif [ "$INSTALL_DEV" = "True" ]; then pip install -e ".[dev]" else pip install -e . @@ -39,6 +52,7 @@ steps: displayName: Install Site Ops env: INSTALL_DEV: ${{ parameters.installDev }} + SITEOPS_SOURCE: ${{ parameters.siteopsSource }} - script: siteops --version displayName: Verify Site Ops installation diff --git a/.pipelines/templates/siteops-deploy.yaml b/.pipelines/templates/siteops-deploy.yaml index 7664eb5..c417bee 100644 --- a/.pipelines/templates/siteops-deploy.yaml +++ b/.pipelines/templates/siteops-deploy.yaml @@ -24,6 +24,10 @@ parameters: displayName: Preview only type: boolean default: false + - name: siteopsSource + displayName: siteops install source (empty = local editable install) + type: string + default: '' stages: - stage: deploy @@ -42,6 +46,9 @@ stages: persistCredentials: false - template: setup-siteops.yaml + parameters: + siteopsSource: ${{ parameters.siteopsSource }} + enableCache: false - script: | # Prevent path traversal attacks @@ -69,13 +76,13 @@ stages: - script: | # Generate sites.local/ override files from SITE_OVERRIDES JSON secret. - # Supports nested paths using dot notation for parameters: + # Supports nested paths using dot notation at any depth: # { # "munich-dev": { # "subscription": "...", # "resourceGroup": "...", # "parameters.clusterName": "actual-cluster-name", - # "parameters.customLocationName": "actual-custom-location" + # "parameters.dataflowIdentity.clientId": "..." # } # } @@ -84,39 +91,38 @@ stages: exit 0 fi - if ! echo "$SITE_OVERRIDES" | jq empty 2>/dev/null; then - echo "##vso[task.logissue type=error]SITE_OVERRIDES secret is not valid JSON" - exit 1 - fi + python3 - <<'GENERATE_OVERLAYS' + import json, os, re, sys, yaml - mkdir -p "$WORKSPACE/sites.local" + overrides = json.loads(os.environ["SITE_OVERRIDES"]) + workspace = os.environ["WORKSPACE"] + site_name_re = re.compile(r"^[a-zA-Z0-9_-]+$") - for site_name in $(echo "$SITE_OVERRIDES" | jq -r 'keys[]'); do - # Sanitize site name - only allow alphanumeric, dash, underscore - if [[ ! "$site_name" =~ ^[a-zA-Z0-9_-]+$ ]]; then - echo "##vso[task.logissue type=error]Invalid site name in SITE_OVERRIDES: $site_name" - exit 1 - fi + sites_local = os.path.join(workspace, "sites.local") + os.makedirs(sites_local, exist_ok=True) - output_file="$WORKSPACE/sites.local/${site_name}.yaml" + for site_name, values in overrides.items(): + if not site_name_re.match(site_name): + print(f"##vso[task.logissue type=error]Invalid site name in SITE_OVERRIDES: {site_name}") + sys.exit(1) - # Build the YAML file with proper nesting - { - # Write top-level keys first - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | to_entries[] | select(.key | contains(".") | not) | "\(.key): \"\(.value)\""' + # Expand dot-notation keys into nested dict + nested = {} + for key, value in values.items(): + parts = key.split(".") + current = nested + for part in parts[:-1]: + current = current.setdefault(part, {}) + current[parts[-1]] = value - # Then write nested keys grouped by parent - for parent in $(echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" \ - '.[$site] | keys[] | select(contains(".")) | split(".")[0]' | sort -u); do - echo "${parent}:" - echo "$SITE_OVERRIDES" | jq -r --arg site "$site_name" --arg parent "$parent" \ - '.[$site] | to_entries[] | select(.key | startswith($parent + ".")) | " \(.key | split(".")[1]): \"\(.value)\""' - done - } > "$output_file" + output_file = os.path.join(sites_local, f"{site_name}.yaml") + with open(output_file, "w") as f: + yaml.safe_dump(nested, f, default_flow_style=False) - echo "✓ Generated override for: $site_name" - done + print(f"✓ Generated override for: {site_name}") + + print(f"✓ Generated {len(overrides)} site override(s)") + GENERATE_OVERLAYS # Mask each override value in ADO logs. # ADO masking is additive — once a value is registered as secret, @@ -124,9 +130,6 @@ stages: echo "$SITE_OVERRIDES" | jq -r '.. | strings' | while IFS= read -r val; do [[ -n "$val" ]] && echo "##vso[task.setvariable variable=masked;issecret=true]${val}" done - - count=$(ls "$WORKSPACE/sites.local/"*.yaml 2>/dev/null | wc -l) - echo "✓ Generated ${count} site override(s)" displayName: Setup site overrides env: WORKSPACE: ${{ parameters.workspace }}