python.yaml

# Copyright 2024 Zaphiro Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Python Lint & Test

concurrency:
  group: ${{github.repository}}-${{ github.ref_name }}-python
  cancel-in-progress: true

# Permissions needed
# permissions:
#  contents: read
#  packages: write
#  pull-requests: read
#  id-token: write

on:
  workflow_call:
    inputs:
      code-scan:
        required: false
        default: true
        type: boolean
        description: False to skip code-scan exec
      vulnerability-scan:
        required: false
        default: true
        type: boolean
        description: False to skip vulnerability-scan exec
      vulnerability-scan-skip-dirs:
        required: false
        default: "vendor,.github"
        type: string
        description: Comma separated list of directories where vulnerability check is skipped
      pull-dvc-datasets:
        required: false
        default: false
        type: boolean
        description: True to install and configure dvc
      upload-artefacts:
        required: false
        default: false
        type: boolean
        description: True to upload files build by test
      upload-artefacts-dir:
        required: false
        default: "output"
        type: string
        description: Directory with the artifacts to upload
      repositories:
        required: false
        type: string
        default: ""
        description: Comma or newline-separated list of repositories to grant access to during the golang build.
jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      files_changed: ${{ steps.filter.outputs.python == 'true' }}
    steps:
      - uses: actions/checkout@v6
      - uses: dorny/paths-filter@v4
        id: filter
        with:
          filters: |
            python:
              - '*.py'
              - '**/*.py'
              - '.github/workflows/python.yaml'
              - '.docker/*.yml'
              - '.docker/*.yaml'
              - 'poetry.lock'
              - 'pyproject.toml'
  lint:
    # Check if the PR is not from a fork
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.files_changed == 'true' && github.event.pull_request && github.event.pull_request.head.repo.full_name == github.repository }}
    steps:
      - uses: actions/create-github-app-token@v3
        id: generate-token
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_SECRET }}
          repositories: ${{ github.event.repository.name }}${{ inputs.repositories && format(',{0}', inputs.repositories) || '' }}
          permission-contents: write
      - uses: actions/checkout@v6
        with:
          token: ${{ steps.generate-token.outputs.token }}
          ref: ${{github.event.pull_request.head.ref}}
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v6
        with:
          python-version-file: "pyproject.toml"
      - name: Install poetry
        run: pipx install poetry==2.3.2 || pipx upgrade poetry
        env:
          PIPX_DEFAULT_PYTHON: ${{ steps.setup-python.outputs.python-path }}
      - name: Cache Poetry artifacts
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/pypoetry/artifacts
            ~/.cache/pip
          key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
          restore-keys: |
            poetry-${{ runner.os }}-
      - name: Install packages
        run: |
          make lint-setup
      - name: Lint
        run: |
          make lint
      - name: Commit lint changes
        run: |
          git config --global user.name 'Bot'
          git config --global user.email 'bot@zaphiro.ch'
          git commit -am "Automated lint fixes [dependabot skip]" || echo "No changes to commit"
          git push
  test:
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ needs.changes.outputs.files_changed == 'true' }}
    steps:
      - uses: actions/create-github-app-token@v3
        id: generate-token
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_SECRET }}
          repositories: ${{ github.event.repository.name }}${{ inputs.repositories && format(',{0}', inputs.repositories) || '' }}
          permission-contents: read
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        # we log in the registry to code check also images
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/checkout@v6
        with:
          token: ${{ steps.generate-token.outputs.token }}
          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v6
        with:
          python-version-file: "pyproject.toml"
      - name: Install poetry
        run: pipx install poetry==2.3.2 || pipx upgrade poetry
        env:
          PIPX_DEFAULT_PYTHON: ${{ steps.setup-python.outputs.python-path }}
      - name: Cache Poetry artifacts
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/pypoetry/artifacts
            ~/.cache/pip
          key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
          restore-keys: |
            poetry-${{ runner.os }}-
      - name: Install packages
        run: |
          make dev-setup
      - name: Configure AWS Credentials
        if: ${{ inputs.pull-dvc-datasets }}
        id: creds
        uses: aws-actions/configure-aws-credentials@v6
        with:
          aws-region: eu-south-1
          role-to-assume: arn:aws:iam::058264335177:role/zaphiro-github-role-demo
          output-credentials: true
      - name: Install DVC
        if: ${{ inputs.pull-dvc-datasets }}
        run: |
          pipx install 'dvc[s3]==3.67.0'
      - name: Set-up credentials
        if: ${{ inputs.pull-dvc-datasets }}
        env:
          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
        run: |
          gh auth setup-git
          git config --global credential.helper '!gh auth git-credential'
      - name: Run tests
        env:
          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
        run: |
          make ci-test
      - if: ${{ inputs.upload-artefacts }}
        name: Upload artefacts
        uses: actions/upload-artifact@v7
        with:
          name: artefacts
          path: ${{ inputs.upload-artefacts-dir }}
          if-no-files-found: error
      - name: SonarCloud Code Scan
        if: ${{ inputs.code-scan }}
        uses: SonarSource/sonarqube-scan-action@v8.0.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
  vulnerability:
    needs: changes
    if: ${{ inputs.vulnerability-scan}}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Run Trivy vulnerability scanner (for summary)
        # trivy-action v0.35.0 (safe version)
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25
        with:
          scan-type: fs
          scan-ref: .
          skip-dirs: ${{ inputs.vulnerability-scan-skip-dirs }}
          scanners: "vuln,secret,misconfig"
          hide-progress: true
          output: trivy.txt
        env:
          TRIVY_SKIP_DB_UPDATE: true
          TRIVY_SKIP_JAVA_DB_UPDATE: true
      - name: Publish Trivy Output to Summary
        run: |
          if [[ -s trivy.txt ]]; then
            {
              echo "### Trivy Security Output"
              echo ""
              echo '```text'
              cat trivy.txt
              echo '```'
            } >> $GITHUB_STEP_SUMMARY
          fi
      # this comes as last or scan results won't be uploaded
      - name: Run Trivy vulnerability scanner (security treshold)
        # trivy-action v0.35.0 (safe version)
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25
        with:
          scan-type: fs
          scan-ref: .
          skip-dirs: ${{ inputs.vulnerability-scan-skip-dirs }}
          scanners: "vuln,secret,misconfig"
          exit-code: "1"
          severity: MEDIUM,HIGH,CRITICAL
          # On a subsequent call to the action we know trivy is already installed so can skip this
          skip-setup-trivy: true
        env:
          TRIVY_SKIP_DB_UPDATE: true
          TRIVY_SKIP_JAVA_DB_UPDATE: true