From 11bbdeac1a9cd6688396c1afb5c299dc6ca660a6 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 14:59:31 +0200 Subject: [PATCH 01/90] Add a security scanning workflow --- .github/workflows/security-scan.yml | 105 ++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 .github/workflows/security-scan.yml diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 00000000..fdad5c39 --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,105 @@ +name: Security Scanning + +on: + # Trigger 1: PR created on main or version branches (*.*) + pull_request: + branches: + - main + - '[0-9]+.[0-9]+' + types: [opened, edited, reopened, synchronize] + + # Trigger 2: Daily scheduled run at 22:00 UTC + schedule: + - cron: '0 22 * * *' + + # Allow manual triggering for testing + workflow_dispatch: + +jobs: + check-recent-activity: + runs-on: ubuntu-latest + if: github.event_name == 'schedule' + outputs: + branches-to-scan: ${{ steps.filter-branches.outputs.branches }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get target branches + id: get-branches + run: | + # Get main branch and all version branches (*.*) + branches=$(git branch -r | grep -E 'origin/(main|[0-9]+\.[0-9]+)$' | sed 's/origin\///' | tr '\n' ' ') + echo "branches=$branches" >> $GITHUB_OUTPUT + echo "Found branches: $branches" + + - name: Filter branches based on recent PR activity + id: filter-branches + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + branches="${{ steps.get-branches.outputs.branches }}" + branches_to_scan="" + + # Check each branch for PRs created in the last 24 hours + for branch in $branches; do + echo "Checking branch: $branch" + + # Get PRs created in the last 24 hours for this branch + recent_prs=$(gh pr list \ + --base "$branch" \ + --state all \ + --json createdAt,headRefName \ + --jq --arg since "$(date -d '24 hours ago' -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.[] | select(.createdAt > $since)') + + if [ -n "$recent_prs" ]; then + echo "Skipping branch $branch - found recent PR activity in the last 24 hours" + echo "Recent PRs: $recent_prs" + else + echo "Adding branch $branch to scan list - no recent PR activity" + branches_to_scan="$branches_to_scan $branch" + fi + done + + # Clean up extra spaces and convert to JSON array format + branches_to_scan=$(echo $branches_to_scan | xargs) + if [ -n "$branches_to_scan" ]; then + # Convert space-separated list to JSON array + json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s .) + echo "branches=$json_branches" >> $GITHUB_OUTPUT + echo "Branches to scan: $json_branches" + else + echo "branches=[]" >> $GITHUB_OUTPUT + echo "No branches to scan - all have recent PR activity" + fi + + security-scan: + runs-on: ubuntu-latest + needs: [check-recent-activity] + if: | + github.event_name == 'pull_request' || + (github.event_name == 'schedule' && needs.check-recent-activity.outputs.branches-to-scan != '[]') || + github.event_name == 'workflow_dispatch' + strategy: + matrix: + branch: ${{ github.event_name == 'schedule' && fromJson(needs.check-recent-activity.outputs.branches-to-scan) || fromJson('["current"]') }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'schedule' && matrix.branch || github.ref }} + + - name: Security Scanning Placeholder + run: | + echo "Security Scanning Started" + echo "Branch: ${{ github.event_name == 'schedule' && matrix.branch || github.ref_name }}" + echo "Trigger: ${{ github.event_name }}" + echo "Repository: ${{ github.repository }}" + echo "Timestamp: $(date -u)" + + # Placeholder for actual security scanning logic + # This is where you'll add your security scanning tools and commands + echo "TODO: Add security scanning implementation" \ No newline at end of file From 865551f332a73881fcdf2bed692e44939542e80f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 15:12:46 +0200 Subject: [PATCH 02/90] Update workflow to run based on past workflow runs instead of past PRs --- .github/workflows/security-scan.yml | 40 ++++++++++++++++------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index fdad5c39..079d7d91 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -16,7 +16,7 @@ on: workflow_dispatch: jobs: - check-recent-activity: + check-recent-workflow-runs: runs-on: ubuntu-latest if: github.event_name == 'schedule' outputs: @@ -35,31 +35,35 @@ jobs: echo "branches=$branches" >> $GITHUB_OUTPUT echo "Found branches: $branches" - - name: Filter branches based on recent PR activity + - name: Filter branches based on recent workflow runs id: filter-branches env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | branches="${{ steps.get-branches.outputs.branches }}" branches_to_scan="" + workflow_name="Security Scanning" + since_date=$(date -d '24 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) - # Check each branch for PRs created in the last 24 hours + echo "Checking for workflow runs since: $since_date" + + # Check each branch for recent workflow runs for branch in $branches; do echo "Checking branch: $branch" - # Get PRs created in the last 24 hours for this branch - recent_prs=$(gh pr list \ - --base "$branch" \ - --state all \ - --json createdAt,headRefName \ - --jq --arg since "$(date -d '24 hours ago' -u +%Y-%m-%dT%H:%M:%SZ)" \ - '.[] | select(.createdAt > $since)') + # Get recent workflow runs for this specific workflow and branch + recent_runs=$(gh run list \ + --workflow="$workflow_name" \ + --branch="$branch" \ + --json createdAt,conclusion,headBranch \ + --jq --arg since "$since_date" \ + '.[] | select(.createdAt > $since and .headBranch == "'$branch'")') - if [ -n "$recent_prs" ]; then - echo "Skipping branch $branch - found recent PR activity in the last 24 hours" - echo "Recent PRs: $recent_prs" + if [ -n "$recent_runs" ]; then + echo "Skipping branch $branch - found recent workflow run in the last 24 hours" + echo "Recent runs: $recent_runs" else - echo "Adding branch $branch to scan list - no recent PR activity" + echo "Adding branch $branch to scan list - no recent workflow runs" branches_to_scan="$branches_to_scan $branch" fi done @@ -73,19 +77,19 @@ jobs: echo "Branches to scan: $json_branches" else echo "branches=[]" >> $GITHUB_OUTPUT - echo "No branches to scan - all have recent PR activity" + echo "No branches to scan - all have recent workflow runs" fi security-scan: runs-on: ubuntu-latest - needs: [check-recent-activity] + needs: [check-recent-workflow-runs] if: | github.event_name == 'pull_request' || - (github.event_name == 'schedule' && needs.check-recent-activity.outputs.branches-to-scan != '[]') || + (github.event_name == 'schedule' && needs.check-recent-workflow-runs.outputs.branches-to-scan != '[]') || github.event_name == 'workflow_dispatch' strategy: matrix: - branch: ${{ github.event_name == 'schedule' && fromJson(needs.check-recent-activity.outputs.branches-to-scan) || fromJson('["current"]') }} + branch: ${{ github.event_name == 'schedule' && fromJson(needs.check-recent-workflow-runs.outputs.branches-to-scan) || fromJson('["current"]') }} steps: - name: Checkout repository uses: actions/checkout@v4 From 0713c1683db7f5a1f86197562db6e64c509517e6 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 16:41:09 +0200 Subject: [PATCH 03/90] Add a trigger to run the workflow every 5 mins --- .github/workflows/security-scan.yml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 079d7d91..87d79348 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -8,12 +8,9 @@ on: - '[0-9]+.[0-9]+' types: [opened, edited, reopened, synchronize] - # Trigger 2: Daily scheduled run at 22:00 UTC + # Trigger 2: Every 5 minutes on Monday for testing schedule: - - cron: '0 22 * * *' - - # Allow manual triggering for testing - workflow_dispatch: + - cron: '*/5 * * * 1' jobs: check-recent-workflow-runs: @@ -84,22 +81,23 @@ jobs: runs-on: ubuntu-latest needs: [check-recent-workflow-runs] if: | - github.event_name == 'pull_request' || - (github.event_name == 'schedule' && needs.check-recent-workflow-runs.outputs.branches-to-scan != '[]') || - github.event_name == 'workflow_dispatch' + always() && ( + github.event_name == 'pull_request' || + (github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && needs.check-recent-workflow-runs.outputs.branches-to-scan != '[]') + ) strategy: matrix: - branch: ${{ github.event_name == 'schedule' && fromJson(needs.check-recent-workflow-runs.outputs.branches-to-scan) || fromJson('["current"]') }} + branch: ${{ github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && fromJson(needs.check-recent-workflow-runs.outputs.branches-to-scan) || fromJson('["current"]') }} steps: - name: Checkout repository uses: actions/checkout@v4 with: - ref: ${{ github.event_name == 'schedule' && matrix.branch || github.ref }} + ref: ${{ github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && matrix.branch || github.ref }} - name: Security Scanning Placeholder run: | echo "Security Scanning Started" - echo "Branch: ${{ github.event_name == 'schedule' && matrix.branch || github.ref_name }}" + echo "Branch: ${{ github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && matrix.branch || github.ref_name }}" echo "Trigger: ${{ github.event_name }}" echo "Repository: ${{ github.repository }}" echo "Timestamp: $(date -u)" From ed1576b54bc5cc59a13dfb10f6ce58980675ee2c Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 16:50:16 +0200 Subject: [PATCH 04/90] Update trigger --- .github/workflows/security-scan.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 87d79348..ac086f23 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -8,9 +8,9 @@ on: - '[0-9]+.[0-9]+' types: [opened, edited, reopened, synchronize] - # Trigger 2: Every 5 minutes on Monday for testing + # Trigger 2: Every 3 minutes on Monday for testing schedule: - - cron: '*/5 * * * 1' + - cron: '0/3 * * * 1' jobs: check-recent-workflow-runs: @@ -53,6 +53,7 @@ jobs: --workflow="$workflow_name" \ --branch="$branch" \ --json createdAt,conclusion,headBranch \ + --status completed \ --jq --arg since "$since_date" \ '.[] | select(.createdAt > $since and .headBranch == "'$branch'")') From d656edd48402a665ce79d15af74f68bedffc9d6a Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 17:00:21 +0200 Subject: [PATCH 05/90] Update trigger and API call --- .github/workflows/security-scan.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ac086f23..541acf6c 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -8,9 +8,9 @@ on: - '[0-9]+.[0-9]+' types: [opened, edited, reopened, synchronize] - # Trigger 2: Every 3 minutes on Monday for testing + # Trigger 2: Every 5 minutes on Monday for testing schedule: - - cron: '0/3 * * * 1' + - cron: '0/5 * * * 1' jobs: check-recent-workflow-runs: @@ -52,10 +52,10 @@ jobs: recent_runs=$(gh run list \ --workflow="$workflow_name" \ --branch="$branch" \ - --json createdAt,conclusion,headBranch \ + --json startedAt,conclusion,headBranch \ --status completed \ --jq --arg since "$since_date" \ - '.[] | select(.createdAt > $since and .headBranch == "'$branch'")') + '.[] | select(.startedAt > $since and .headBranch == "'$branch'")') if [ -n "$recent_runs" ]; then echo "Skipping branch $branch - found recent workflow run in the last 24 hours" From 02d20e4dfbc6b7a038727807424cce52fbc7bc83 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 17:08:55 +0200 Subject: [PATCH 06/90] Remove automatic trigger for build action --- .github/workflows/build-targets.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-targets.yaml b/.github/workflows/build-targets.yaml index bc5b3b54..e2cce736 100644 --- a/.github/workflows/build-targets.yaml +++ b/.github/workflows/build-targets.yaml @@ -1,9 +1,6 @@ name: Build Code Editor Targets on: - push: - branches: - - 'main' - - '*.*' + workflow_dispatch: jobs: build: From 00015d86401541ae7c2cf5b745a163d90223958e Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 17:23:06 +0200 Subject: [PATCH 07/90] Add back workflow_dispatch trigger --- .github/workflows/security-scan.yml | 47 ++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 541acf6c..ca387573 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -8,16 +8,37 @@ on: - '[0-9]+.[0-9]+' types: [opened, edited, reopened, synchronize] - # Trigger 2: Every 5 minutes on Monday for testing + # Trigger 2: Daily scheduled run at 22:00 UTC schedule: - - cron: '0/5 * * * 1' + - cron: '0 22 * * *' + + workflow_dispatch: jobs: - check-recent-workflow-runs: + security-scan-pr: runs-on: ubuntu-latest - if: github.event_name == 'schedule' + if: github.event_name == 'pull_request' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Security Scanning Placeholder + run: | + echo "Security Scanning Started" + echo "Branch: ${{ github.ref_name }}" + echo "Trigger: ${{ github.event_name }}" + echo "Repository: ${{ github.repository }}" + echo "Timestamp: $(date -u)" + + # Placeholder for actual security scanning logic + # This is where you'll add your security scanning tools and commands + echo "TODO: Add security scanning implementation" + + get-branches-to-scan: + runs-on: ubuntu-latest + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' outputs: - branches-to-scan: ${{ steps.filter-branches.outputs.branches }} + branches: ${{ steps.filter-branches.outputs.branches }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -78,27 +99,23 @@ jobs: echo "No branches to scan - all have recent workflow runs" fi - security-scan: + security-scan-scheduled: runs-on: ubuntu-latest - needs: [check-recent-workflow-runs] - if: | - always() && ( - github.event_name == 'pull_request' || - (github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && needs.check-recent-workflow-runs.outputs.branches-to-scan != '[]') - ) + needs: [get-branches-to-scan] + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' strategy: matrix: - branch: ${{ github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && fromJson(needs.check-recent-workflow-runs.outputs.branches-to-scan) || fromJson('["current"]') }} + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} steps: - name: Checkout repository uses: actions/checkout@v4 with: - ref: ${{ github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && matrix.branch || github.ref }} + ref: ${{ matrix.branch }} - name: Security Scanning Placeholder run: | echo "Security Scanning Started" - echo "Branch: ${{ github.event_name == 'schedule' && needs.check-recent-workflow-runs.result == 'success' && matrix.branch || github.ref_name }}" + echo "Branch: ${{ matrix.branch }}" echo "Trigger: ${{ github.event_name }}" echo "Repository: ${{ github.repository }}" echo "Timestamp: $(date -u)" From 2875c1be9d080db7daa870e0d13cecd69310b85b Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 17:27:27 +0200 Subject: [PATCH 08/90] Fix quotes --- .github/workflows/security-scan.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ca387573..8ae4dc6b 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -75,7 +75,6 @@ jobs: --branch="$branch" \ --json startedAt,conclusion,headBranch \ --status completed \ - --jq --arg since "$since_date" \ '.[] | select(.startedAt > $since and .headBranch == "'$branch'")') if [ -n "$recent_runs" ]; then From 462a3a153b0d89629c5caa3ae6ad85f071258d81 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 17:50:51 +0200 Subject: [PATCH 09/90] Set the jq query correctly --- .github/workflows/security-scan.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 8ae4dc6b..640db9bc 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -70,12 +70,8 @@ jobs: echo "Checking branch: $branch" # Get recent workflow runs for this specific workflow and branch - recent_runs=$(gh run list \ - --workflow="$workflow_name" \ - --branch="$branch" \ - --json startedAt,conclusion,headBranch \ - --status completed \ - '.[] | select(.startedAt > $since and .headBranch == "'$branch'")') + all_runs=$(gh run list --workflow="$workflow_name" --branch="$branch" --json startedAt,conclusion,headBranch --status success) + recent_runs=$(echo "$all_runs" | jq --arg since "$since_date" --arg branch "$branch" '.[] | select(.startedAt > $since and .headBranch == $branch)') if [ -n "$recent_runs" ]; then echo "Skipping branch $branch - found recent workflow run in the last 24 hours" From 7d60e7e48fafed7161016a01284daa0e65fd0cbf Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 18:13:58 +0200 Subject: [PATCH 10/90] Fix JSON formatting --- .github/workflows/security-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 640db9bc..651aa510 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -85,8 +85,8 @@ jobs: # Clean up extra spaces and convert to JSON array format branches_to_scan=$(echo $branches_to_scan | xargs) if [ -n "$branches_to_scan" ]; then - # Convert space-separated list to JSON array - json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s .) + # Convert space-separated list to JSON array (compact format) + json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) echo "branches=$json_branches" >> $GITHUB_OUTPUT echo "Branches to scan: $json_branches" else From 1527a982e89840e19d91f61e9529ad536af146de Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 18:17:14 +0200 Subject: [PATCH 11/90] Change schedule --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 651aa510..70e79d94 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -10,7 +10,7 @@ on: # Trigger 2: Daily scheduled run at 22:00 UTC schedule: - - cron: '0 22 * * *' + - cron: '30 16 * * *' workflow_dispatch: From 88fc314f2c15203c152984e3c36eb4a0a73cc11b Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 19:16:01 +0200 Subject: [PATCH 12/90] Reset the cron timer --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 70e79d94..651aa510 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -10,7 +10,7 @@ on: # Trigger 2: Daily scheduled run at 22:00 UTC schedule: - - cron: '30 16 * * *' + - cron: '0 22 * * *' workflow_dispatch: From bb80784b0dcce571b770f408aa51259b7a998b17 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 19:29:35 +0200 Subject: [PATCH 13/90] Add validation for base branch --- .github/workflows/security-scan.yml | 34 ++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 651aa510..82acfc34 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -5,7 +5,7 @@ on: pull_request: branches: - main - - '[0-9]+.[0-9]+' + - '*.*' types: [opened, edited, reopened, synchronize] # Trigger 2: Daily scheduled run at 22:00 UTC @@ -19,19 +19,43 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'pull_request' steps: - - name: Checkout repository + - name: Validate base branch pattern + run: | + base_ref="${{ github.base_ref }}" + echo "Base branch: $base_ref" + + if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then + echo "Base branch matches allowed pattern (main or digit.digit) - proceeding with security scan" + else + echo "Base branch does not match allowed pattern - skipping security scan" + exit 0 + fi + + - name: Checkout destination branch (for workflow) uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref }} + + - name: Checkout PR branch (for scanning) + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + path: pr-code - name: Security Scanning Placeholder run: | echo "Security Scanning Started" - echo "Branch: ${{ github.ref_name }}" + echo "Base Branch (workflow source): ${{ github.base_ref }}" + echo "PR Branch (code being scanned): ${{ github.head_ref }}" echo "Trigger: ${{ github.event_name }}" echo "Repository: ${{ github.repository }}" echo "Timestamp: $(date -u)" + echo "Scanning code from: ./pr-code/" - # Placeholder for actual security scanning logic - # This is where you'll add your security scanning tools and commands + # The security scanning logic will run from the base branch workflow + # but scan the code checked out in ./pr-code/ directory + echo "TODO: Add security scanning implementation" + echo "Example: scan ./pr-code/ directory for security issues" echo "TODO: Add security scanning implementation" get-branches-to-scan: From e0acb29df7798ea4501e49c57fb09877f221bcf1 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 19:40:34 +0200 Subject: [PATCH 14/90] Adjust environment variable for scanning --- .github/workflows/security-scan.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 82acfc34..5c9f4536 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -25,24 +25,28 @@ jobs: echo "Base branch: $base_ref" if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then - echo "Base branch matches allowed pattern (main or digit.digit) - proceeding with security scan" + echo "Base branch matches allowed pattern (main or digit.digit)" + echo "SHOULD_SCAN=true" >> $GITHUB_ENV else echo "Base branch does not match allowed pattern - skipping security scan" - exit 0 + echo "SHOULD_SCAN=false" >> $GITHUB_ENV fi - name: Checkout destination branch (for workflow) + if: env.SHOULD_SCAN == 'true' uses: actions/checkout@v4 with: ref: ${{ github.base_ref }} - name: Checkout PR branch (for scanning) + if: env.SHOULD_SCAN == 'true' uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} path: pr-code - name: Security Scanning Placeholder + if: env.SHOULD_SCAN == 'true' run: | echo "Security Scanning Started" echo "Base Branch (workflow source): ${{ github.base_ref }}" From 304c0c83e80126c9746842f740e06d2757ad4214 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 21:49:22 +0200 Subject: [PATCH 15/90] Add npm sbom step --- .github/workflows/security-scan.yml | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 5c9f4536..2177ba4a 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -18,9 +18,15 @@ jobs: security-scan-pr: runs-on: ubuntu-latest if: github.event_name == 'pull_request' + strategy: + matrix: + target: [code-editor-server, code-editor-sagemaker-server, code-editor-web-embedded, code-editor-web-embedded-with-terminal] steps: - name: Validate base branch pattern run: | + # PR trigger won't allow regex matching, it only allows *.*. + # So we add an additional validation check here. + base_ref="${{ github.base_ref }}" echo "Base branch: $base_ref" @@ -31,36 +37,30 @@ jobs: echo "Base branch does not match allowed pattern - skipping security scan" echo "SHOULD_SCAN=false" >> $GITHUB_ENV fi - - - name: Checkout destination branch (for workflow) - if: env.SHOULD_SCAN == 'true' - uses: actions/checkout@v4 - with: - ref: ${{ github.base_ref }} - - name: Checkout PR branch (for scanning) + - name: Checkout PR branch if: env.SHOULD_SCAN == 'true' uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} - path: pr-code - - name: Security Scanning Placeholder + - name: Run Security Scan if: env.SHOULD_SCAN == 'true' run: | echo "Security Scanning Started" - echo "Base Branch (workflow source): ${{ github.base_ref }}" + echo "Target: ${{ matrix.target }}" echo "PR Branch (code being scanned): ${{ github.head_ref }}" - echo "Trigger: ${{ github.event_name }}" - echo "Repository: ${{ github.repository }}" - echo "Timestamp: $(date -u)" - echo "Scanning code from: ./pr-code/" + + echo "Preparing source" + ./scripts/prepare-src.sh "${{ matrix.target }}" + + echo "Installing node dependencies" + cd code-editor-src + npm ci - # The security scanning logic will run from the base branch workflow - # but scan the code checked out in ./pr-code/ directory - echo "TODO: Add security scanning implementation" - echo "Example: scan ./pr-code/ directory for security issues" - echo "TODO: Add security scanning implementation" + echo "Generating SBOM" + npm sbom --omit dev --sbom-format cyclonedx --sbom-type application > npm_sbom.json + cat npm_sbom.json get-branches-to-scan: runs-on: ubuntu-latest From 0b5210b0fe6e53b976c92ba29f0295ac21c6732b Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 21:50:03 +0200 Subject: [PATCH 16/90] Test PR --- .github/workflows/security-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 2177ba4a..6c3a7c61 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -24,6 +24,7 @@ jobs: steps: - name: Validate base branch pattern run: | + echo "Hi!" # PR trigger won't allow regex matching, it only allows *.*. # So we add an additional validation check here. From 90ddec4e616e785f2310390bca5bb170c6e201b2 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 21:51:45 +0200 Subject: [PATCH 17/90] Add step to setup build env --- .github/workflows/security-scan.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 2177ba4a..255e6cc3 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -43,6 +43,12 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} + + - name: Set up build environment + run: | + echo "Installing required dependencies" + sudo apt-get update + sudo apt-get install -y quilt - name: Run Security Scan if: env.SHOULD_SCAN == 'true' From 7298d964bd9760fe05089db414e702c7cebea4de Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 21:53:38 +0200 Subject: [PATCH 18/90] Only scan SageMaker server target for now --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 255e6cc3..556f1af3 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -20,7 +20,7 @@ jobs: if: github.event_name == 'pull_request' strategy: matrix: - target: [code-editor-server, code-editor-sagemaker-server, code-editor-web-embedded, code-editor-web-embedded-with-terminal] + target: [code-editor-sagemaker-server] steps: - name: Validate base branch pattern run: | From e2ca1073f919e927b7bbeac2c42dcf12ea46f8aa Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 21:57:23 +0200 Subject: [PATCH 19/90] Add recursive submodule checkout --- .github/workflows/security-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 556f1af3..3e7c4ae8 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -43,6 +43,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} + submodules: recursive - name: Set up build environment run: | From 79b149cf4021e2de2f439925dfb8264c5c906759 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 22:02:56 +0200 Subject: [PATCH 20/90] Add step to setup Node JS --- .github/workflows/security-scan.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 3e7c4ae8..2cf41cfe 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -45,25 +45,36 @@ jobs: ref: ${{ github.head_ref }} submodules: recursive - - name: Set up build environment + - name: Set up environment run: | echo "Installing required dependencies" sudo apt-get update sudo apt-get install -y quilt + - name: Run patches script + run: | + ./scripts/prepare-src.sh ${{ matrix.build-target }} + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: 'code-editor-src/package-lock.json' + + - name: Install Code Editor Dependencies + run: | + cd code-editor-src + echo "Installing Dependencies" + npm ci + - name: Run Security Scan if: env.SHOULD_SCAN == 'true' run: | echo "Security Scanning Started" echo "Target: ${{ matrix.target }}" echo "PR Branch (code being scanned): ${{ github.head_ref }}" - - echo "Preparing source" - ./scripts/prepare-src.sh "${{ matrix.target }}" - - echo "Installing node dependencies" cd code-editor-src - npm ci echo "Generating SBOM" npm sbom --omit dev --sbom-format cyclonedx --sbom-type application > npm_sbom.json From cccdac03393d1b4b9187f349d8c0bb0ec6f25d33 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 22:06:51 +0200 Subject: [PATCH 21/90] Install all packages that are required --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 2cf41cfe..d45301ef 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -49,7 +49,7 @@ jobs: run: | echo "Installing required dependencies" sudo apt-get update - sudo apt-get install -y quilt + sudo apt-get install -y quilt libkrb5-dev libx11-dev libxkbfile-dev libxml2-utils - name: Run patches script run: | From c11b62691ae0d0aeadead6b967d8032b8b59afcd Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 22:20:07 +0200 Subject: [PATCH 22/90] Use CycloneDX --- .github/workflows/security-scan.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index d45301ef..b0c5cf4e 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -74,11 +74,14 @@ jobs: echo "Security Scanning Started" echo "Target: ${{ matrix.target }}" echo "PR Branch (code being scanned): ${{ github.head_ref }}" - cd code-editor-src + + echo "Installing CycloneDX SBOM for npm" + npm i -g @cyclonedx/cyclonedx-npm echo "Generating SBOM" - npm sbom --omit dev --sbom-format cyclonedx --sbom-type application > npm_sbom.json - cat npm_sbom.json + cd code-editor-src + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o scan_sbom.json + cat scan_sbom.json get-branches-to-scan: runs-on: ubuntu-latest From 478239e343f19b78f1f3dbb6f37f6f831b8591d0 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 22:46:44 +0200 Subject: [PATCH 23/90] Add AWS Integration --- .github/workflows/security-scan.yml | 48 +++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index b0c5cf4e..a2aaf234 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -18,6 +18,9 @@ jobs: security-scan-pr: runs-on: ubuntu-latest if: github.event_name == 'pull_request' + environment: security-scanning-workflow-env + permissions: + id-token: write # Required for OIDC strategy: matrix: target: [code-editor-sagemaker-server] @@ -67,6 +70,18 @@ jobs: cd code-editor-src echo "Installing Dependencies" npm ci + + - name: Install Security Scan Dependencies + run: | + echo "Installing CycloneDX SBOM for npm" + npm i -g @cyclonedx/cyclonedx-npm + + - name: Assume IAM Role + id: assume-aws-iam-role + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} + aws-region: us-east-1 - name: Run Security Scan if: env.SHOULD_SCAN == 'true' @@ -74,14 +89,35 @@ jobs: echo "Security Scanning Started" echo "Target: ${{ matrix.target }}" echo "PR Branch (code being scanned): ${{ github.head_ref }}" - - echo "Installing CycloneDX SBOM for npm" - npm i -g @cyclonedx/cyclonedx-npm - + echo "Generating SBOM" cd code-editor-src - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o scan_sbom.json - cat scan_sbom.json + # 1.5 Spec Version compatible with Inspector's ScanSbom API. + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o code-editor-sbom.json + + echo "Invoking Inspector's ScanSbom API" + aws inspector-scan scan-sbom --sbom file://code-editor-sbom.json > sbom_scan_result.json + + echo "Publish success metric for Security Scan" + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "ScanComplete" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --value 1 + + - name: Output SBOM + uses: actions/upload-artifact@v4 + with: + name: code-editor-sbom.json + path: code-editor-src/code-editor-sbom.json + retention-days: 90 + + - name: Output Scan Results + uses: actions/upload-artifact@v4 + with: + name: sbom_scan_result.json + path: code-editor-src/sbom_scan_result.json + retention-days: 90 get-branches-to-scan: runs-on: ubuntu-latest From 081cc719d4713db40237e6d48594fc51c0b79c64 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 23:00:40 +0200 Subject: [PATCH 24/90] Add result analysis for SBOM Scan --- .github/workflows/security-scan.yml | 60 ++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index a2aaf234..a93ae201 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -49,16 +49,19 @@ jobs: submodules: recursive - name: Set up environment + if: env.SHOULD_SCAN == 'true' run: | echo "Installing required dependencies" sudo apt-get update sudo apt-get install -y quilt libkrb5-dev libx11-dev libxkbfile-dev libxml2-utils - name: Run patches script + if: env.SHOULD_SCAN == 'true' run: | ./scripts/prepare-src.sh ${{ matrix.build-target }} - name: Set up Node.js + if: env.SHOULD_SCAN == 'true' uses: actions/setup-node@v4 with: node-version: '22' @@ -66,17 +69,20 @@ jobs: cache-dependency-path: 'code-editor-src/package-lock.json' - name: Install Code Editor Dependencies + if: env.SHOULD_SCAN == 'true' run: | cd code-editor-src echo "Installing Dependencies" npm ci - name: Install Security Scan Dependencies + if: env.SHOULD_SCAN == 'true' run: | echo "Installing CycloneDX SBOM for npm" npm i -g @cyclonedx/cyclonedx-npm - name: Assume IAM Role + if: env.SHOULD_SCAN == 'true' id: assume-aws-iam-role uses: aws-actions/configure-aws-credentials@v4 with: @@ -101,9 +107,61 @@ jobs: echo "Publish success metric for Security Scan" aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ - --metric-name "ScanComplete" \ + --metric-name "SecurityScanInvoked" \ --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ --value 1 + + - name: Analyze SBOM Scan Results + if: env.SHOULD_SCAN == 'true' + run: | + cd code-editor-src + + # Check if scan results file exists + if [ ! -f "sbom_scan_result.json" ]; then + echo "Error: SBOM scan results file not found" + exit 1 + fi + + # Extract vulnerability counts from the scan results + critical=$(jq -r '.sbom.vulnerability_count.critical // 0' sbom_scan_result.json) + high=$(jq -r '.sbom.vulnerability_count.high // 0' sbom_scan_result.json) + medium=$(jq -r '.sbom.vulnerability_count.medium // 0' sbom_scan_result.json) + other=$(jq -r '.sbom.vulnerability_count.other // 0' sbom_scan_result.json) + low=$(jq -r '.sbom.vulnerability_count.low // 0' sbom_scan_result.json) + + echo "=== SBOM Security Scan Results for ${{ matrix.target }} ===" + echo "Critical vulnerabilities: $critical" + echo "High vulnerabilities: $high" + echo "Medium vulnerabilities: $medium" + echo "Other vulnerabilities: $other" + echo "Low vulnerabilities: $low" + echo "==================================================" + + # Calculate total concerning vulnerabilities (excluding low) + total_concerning=$((critical + high + medium + other)) + + if [ $total_concerning -gt 0 ]; then + echo "❌ Security scan FAILED: Found $total_concerning concerning vulnerabilities" + + # Publish failure metric + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanFailed" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning,Target=${{ matrix.target }}" \ + --value 1 + + exit 1 + else + echo "✅ Security scan PASSED: No concerning vulnerabilities found" + echo "Low vulnerabilities: $low (acceptable)" + + # Publish success metric + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanPassed" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning,Target=${{ matrix.target }}" \ + --value 1 + fi - name: Output SBOM uses: actions/upload-artifact@v4 From 63c1167435d9537132136e96507fc804e650fe62 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 23:09:31 +0200 Subject: [PATCH 25/90] Change metric dimensions --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index a93ae201..93c7beea 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -108,7 +108,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning,Target=${{ matrix.target }}" \ --value 1 - name: Analyze SBOM Scan Results From a8c3c656cd8e43d16dd1e9cc48e14c922e97db70 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 23:15:33 +0200 Subject: [PATCH 26/90] Add Security Scanning bash script --- .github/workflows/security-scan.yml | 68 +---------------- scripts/security-scan.sh | 111 ++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 66 deletions(-) create mode 100755 scripts/security-scan.sh diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 93c7beea..a512ea00 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -92,76 +92,12 @@ jobs: - name: Run Security Scan if: env.SHOULD_SCAN == 'true' run: | - echo "Security Scanning Started" - echo "Target: ${{ matrix.target }}" - echo "PR Branch (code being scanned): ${{ github.head_ref }}" - - echo "Generating SBOM" - cd code-editor-src - # 1.5 Spec Version compatible with Inspector's ScanSbom API. - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o code-editor-sbom.json - - echo "Invoking Inspector's ScanSbom API" - aws inspector-scan scan-sbom --sbom file://code-editor-sbom.json > sbom_scan_result.json - - echo "Publish success metric for Security Scan" - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning,Target=${{ matrix.target }}" \ - --value 1 + ./scripts/security-scan.sh run-scan "${{ matrix.target }}" "${{ github.repository }}" "${{ github.head_ref }}" - name: Analyze SBOM Scan Results if: env.SHOULD_SCAN == 'true' run: | - cd code-editor-src - - # Check if scan results file exists - if [ ! -f "sbom_scan_result.json" ]; then - echo "Error: SBOM scan results file not found" - exit 1 - fi - - # Extract vulnerability counts from the scan results - critical=$(jq -r '.sbom.vulnerability_count.critical // 0' sbom_scan_result.json) - high=$(jq -r '.sbom.vulnerability_count.high // 0' sbom_scan_result.json) - medium=$(jq -r '.sbom.vulnerability_count.medium // 0' sbom_scan_result.json) - other=$(jq -r '.sbom.vulnerability_count.other // 0' sbom_scan_result.json) - low=$(jq -r '.sbom.vulnerability_count.low // 0' sbom_scan_result.json) - - echo "=== SBOM Security Scan Results for ${{ matrix.target }} ===" - echo "Critical vulnerabilities: $critical" - echo "High vulnerabilities: $high" - echo "Medium vulnerabilities: $medium" - echo "Other vulnerabilities: $other" - echo "Low vulnerabilities: $low" - echo "==================================================" - - # Calculate total concerning vulnerabilities (excluding low) - total_concerning=$((critical + high + medium + other)) - - if [ $total_concerning -gt 0 ]; then - echo "❌ Security scan FAILED: Found $total_concerning concerning vulnerabilities" - - # Publish failure metric - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning,Target=${{ matrix.target }}" \ - --value 1 - - exit 1 - else - echo "✅ Security scan PASSED: No concerning vulnerabilities found" - echo "Low vulnerabilities: $low (acceptable)" - - # Publish success metric - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-name "SecurityScanPassed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning,Target=${{ matrix.target }}" \ - --value 1 - fi + ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" - name: Output SBOM uses: actions/upload-artifact@v4 diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh new file mode 100755 index 00000000..9726985a --- /dev/null +++ b/scripts/security-scan.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +set -e + +# Function to run the SBOM security scan +run_security_scan() { + local target="$1" + local repository="$2" + local head_ref="$3" + + echo "Security Scanning Started" + echo "Target: $target" + echo "PR Branch (code being scanned): $head_ref" + + echo "Generating SBOM" + cd code-editor-src + # 1.5 Spec Version compatible with Inspector's ScanSbom API. + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o code-editor-sbom.json + + echo "Invoking Inspector's ScanSbom API" + aws inspector-scan scan-sbom --sbom file://code-editor-sbom.json > sbom_scan_result.json + + echo "Publish success metric for Security Scan" + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "ScanComplete" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning" \ + --value 1 +} + +# Function to analyze SBOM scan results +analyze_sbom_results() { + local target="$1" + local repository="$2" + + cd code-editor-src + + # Check if scan results file exists + if [ ! -f "sbom_scan_result.json" ]; then + echo "Error: SBOM scan results file not found" + exit 1 + fi + + # Extract vulnerability counts from the scan results + critical=$(jq -r '.sbom.vulnerability_count.critical // 0' sbom_scan_result.json) + high=$(jq -r '.sbom.vulnerability_count.high // 0' sbom_scan_result.json) + medium=$(jq -r '.sbom.vulnerability_count.medium // 0' sbom_scan_result.json) + other=$(jq -r '.sbom.vulnerability_count.other // 0' sbom_scan_result.json) + low=$(jq -r '.sbom.vulnerability_count.low // 0' sbom_scan_result.json) + + echo "=== SBOM Security Scan Results for $target ===" + echo "Critical vulnerabilities: $critical" + echo "High vulnerabilities: $high" + echo "Medium vulnerabilities: $medium" + echo "Other vulnerabilities: $other" + echo "Low vulnerabilities: $low" + echo "==================================================" + + # Calculate total concerning vulnerabilities (excluding low) + total_concerning=$((critical + high + medium + other)) + + if [ $total_concerning -gt 0 ]; then + echo "❌ Security scan FAILED: Found $total_concerning concerning vulnerabilities" + echo "Critical: $critical, High: $high, Medium: $medium, Other: $other" + + # Display detailed vulnerability messages if available + echo "" + echo "Vulnerability details:" + jq -r '.sbom.messages[]? | select(.vulnerability_message or .error_message) | "- \(.purl // "Unknown"): \(.vulnerability_message // .error_message // .info_message)"' sbom_scan_result.json || echo "No detailed messages available" + + # Publish failure metric + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanFailed" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning,Target=$target" \ + --value $total_concerning + + exit 1 + else + echo "✅ Security scan PASSED: No concerning vulnerabilities found" + echo "Low vulnerabilities: $low (acceptable)" + + # Publish success metric + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanPassed" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning,Target=$target" \ + --value 1 + fi +} + +# Main function to handle command line arguments +main() { + case "$1" in + "run-scan") + run_security_scan "$2" "$3" "$4" + ;; + "analyze-results") + analyze_sbom_results "$2" "$3" + ;; + *) + echo "Usage: $0 {run-scan|analyze-results} [head_ref]" + echo " run-scan: Execute the SBOM security scan" + echo " analyze-results: Analyze SBOM scan results and fail if vulnerabilities found" + exit 1 + ;; + esac +} + +# Call main function with all arguments +main "$@" \ No newline at end of file From 1d64d7dc7cdbe735755fb84188a74f1be2bbdae1 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 23:16:03 +0200 Subject: [PATCH 27/90] Correct shebang --- scripts/security-scan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 9726985a..9442f8d9 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e From 007e25c2bee73e499eea433e4873124a3a24237e Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 23:23:21 +0200 Subject: [PATCH 28/90] Install tar-fs --- .github/workflows/security-scan.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ded5584f..293c5a20 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -75,6 +75,7 @@ jobs: cd code-editor-src echo "Installing Dependencies" npm ci + npm install tar-fs@2.0.0 - name: Install Security Scan Dependencies if: env.SHOULD_SCAN == 'true' @@ -88,7 +89,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} - aws-region: us-east-1 + aws-region: us-east-1 - name: Run Security Scan if: env.SHOULD_SCAN == 'true' From 5b0903c3f33e712b57914e5261415c5a0ceddfeb Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 23:31:17 +0200 Subject: [PATCH 29/90] Change order of outputting scan results --- .github/workflows/security-scan.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index a512ea00..5bdf3e8b 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -94,11 +94,6 @@ jobs: run: | ./scripts/security-scan.sh run-scan "${{ matrix.target }}" "${{ github.repository }}" "${{ github.head_ref }}" - - name: Analyze SBOM Scan Results - if: env.SHOULD_SCAN == 'true' - run: | - ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" - - name: Output SBOM uses: actions/upload-artifact@v4 with: @@ -113,6 +108,11 @@ jobs: path: code-editor-src/sbom_scan_result.json retention-days: 90 + - name: Analyze SBOM Scan Results + if: env.SHOULD_SCAN == 'true' + run: | + ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" + get-branches-to-scan: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' From a57a4faf5c52b7b201d13bdc1f79143f74e29d52 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 25 Aug 2025 23:43:38 +0200 Subject: [PATCH 30/90] Add scanned results directories --- scripts/security-scan.sh | 143 ++++++++++++++++++++++++++++++--------- 1 file changed, 111 insertions(+), 32 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 9442f8d9..49ad6627 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -12,13 +12,51 @@ run_security_scan() { echo "Target: $target" echo "PR Branch (code being scanned): $head_ref" - echo "Generating SBOM" - cd code-editor-src - # 1.5 Spec Version compatible with Inspector's ScanSbom API. - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o code-editor-sbom.json + # Define directories to scan + local scan_dirs=("code-editor-src" "code-editor-src/remote" "code-editor-src/extensions" "code-editor-src/remote/web") + local scan_results=() - echo "Invoking Inspector's ScanSbom API" - aws inspector-scan scan-sbom --sbom file://code-editor-sbom.json > sbom_scan_result.json + # Scan each directory + for dir in "${scan_dirs[@]}"; do + echo "=== Scanning directory: $dir ===" + + # Check if directory exists and has package.json + if [ ! -d "$dir" ]; then + echo "Warning: Directory $dir does not exist, skipping..." + continue + fi + + if [ ! -f "$dir/package.json" ]; then + echo "Warning: No package.json found in $dir, skipping..." + continue + fi + + # Generate SBOM for this directory + echo "Generating SBOM for $dir" + cd "$dir" + + # Create a safe filename for the SBOM + local safe_dir_name=$(echo "$dir" | sed 's/\//_/g') + local sbom_file="${safe_dir_name}-sbom.json" + local result_file="${safe_dir_name}-scan-result.json" + + # 1.5 Spec Version compatible with Inspector's ScanSbom API + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" + + echo "Invoking Inspector's ScanSbom API for $dir" + aws inspector-scan scan-sbom --sbom "file://$sbom_file" > "$result_file" + + # Store the result file path for later analysis + scan_results+=("$PWD/$result_file") + + # Return to root directory + cd - > /dev/null + + echo "Completed scan for $dir" + done + + # Store scan results paths in a file for the analyze step + printf '%s\n' "${scan_results[@]}" > scan_results_paths.txt echo "Publish success metric for Security Scan" aws cloudwatch put-metric-data \ @@ -33,40 +71,81 @@ analyze_sbom_results() { local target="$1" local repository="$2" - cd code-editor-src - - # Check if scan results file exists - if [ ! -f "sbom_scan_result.json" ]; then - echo "Error: SBOM scan results file not found" + # Check if scan results paths file exists + if [ ! -f "scan_results_paths.txt" ]; then + echo "Error: Scan results paths file not found" exit 1 fi - # Extract vulnerability counts from the scan results - critical=$(jq -r '.sbom.vulnerability_count.critical // 0' sbom_scan_result.json) - high=$(jq -r '.sbom.vulnerability_count.high // 0' sbom_scan_result.json) - medium=$(jq -r '.sbom.vulnerability_count.medium // 0' sbom_scan_result.json) - other=$(jq -r '.sbom.vulnerability_count.other // 0' sbom_scan_result.json) - low=$(jq -r '.sbom.vulnerability_count.low // 0' sbom_scan_result.json) + # Initialize totals + local total_critical=0 + local total_high=0 + local total_medium=0 + local total_other=0 + local total_low=0 + local has_failures=false echo "=== SBOM Security Scan Results for $target ===" - echo "Critical vulnerabilities: $critical" - echo "High vulnerabilities: $high" - echo "Medium vulnerabilities: $medium" - echo "Other vulnerabilities: $other" - echo "Low vulnerabilities: $low" + + # Process each scan result file + while IFS= read -r result_file; do + if [ ! -f "$result_file" ]; then + echo "Warning: Scan result file $result_file not found, skipping..." + continue + fi + + # Extract directory name from result file path + local dir_name=$(basename "$result_file" | sed 's/-scan-result\.json$//' | sed 's/_/\//g') + + echo "" + echo "--- Results for $dir_name ---" + + # Extract vulnerability counts from this scan result + local critical=$(jq -r '.sbom.vulnerability_count.critical // 0' "$result_file") + local high=$(jq -r '.sbom.vulnerability_count.high // 0' "$result_file") + local medium=$(jq -r '.sbom.vulnerability_count.medium // 0' "$result_file") + local other=$(jq -r '.sbom.vulnerability_count.other // 0' "$result_file") + local low=$(jq -r '.sbom.vulnerability_count.low // 0' "$result_file") + + echo "Critical: $critical, High: $high, Medium: $medium, Other: $other, Low: $low" + + # Add to totals + total_critical=$((total_critical + critical)) + total_high=$((total_high + high)) + total_medium=$((total_medium + medium)) + total_other=$((total_other + other)) + total_low=$((total_low + low)) + + # Check for concerning vulnerabilities in this directory + local dir_concerning=$((critical + high + medium + other)) + if [ $dir_concerning -gt 0 ]; then + has_failures=true + echo "⚠️ Found $dir_concerning concerning vulnerabilities in $dir_name" + + # Display detailed vulnerability messages if available + echo "Vulnerability details for $dir_name:" + jq -r '.sbom.messages[]? | select(.vulnerability_message or .error_message) | " - \(.purl // "Unknown"): \(.vulnerability_message // .error_message // .info_message)"' "$result_file" || echo " No detailed messages available" + else + echo "✅ No concerning vulnerabilities in $dir_name" + fi + + done < scan_results_paths.txt + + echo "" + echo "=== TOTAL SCAN RESULTS ===" + echo "Total Critical vulnerabilities: $total_critical" + echo "Total High vulnerabilities: $total_high" + echo "Total Medium vulnerabilities: $total_medium" + echo "Total Other vulnerabilities: $total_other" + echo "Total Low vulnerabilities: $total_low" echo "==================================================" # Calculate total concerning vulnerabilities (excluding low) - total_concerning=$((critical + high + medium + other)) + local total_concerning=$((total_critical + total_high + total_medium + total_other)) if [ $total_concerning -gt 0 ]; then - echo "❌ Security scan FAILED: Found $total_concerning concerning vulnerabilities" - echo "Critical: $critical, High: $high, Medium: $medium, Other: $other" - - # Display detailed vulnerability messages if available - echo "" - echo "Vulnerability details:" - jq -r '.sbom.messages[]? | select(.vulnerability_message or .error_message) | "- \(.purl // "Unknown"): \(.vulnerability_message // .error_message // .info_message)"' sbom_scan_result.json || echo "No detailed messages available" + echo "❌ Security scan FAILED: Found $total_concerning concerning vulnerabilities across all directories" + echo "Critical: $total_critical, High: $total_high, Medium: $total_medium, Other: $total_other" # Publish failure metric aws cloudwatch put-metric-data \ @@ -77,8 +156,8 @@ analyze_sbom_results() { exit 1 else - echo "✅ Security scan PASSED: No concerning vulnerabilities found" - echo "Low vulnerabilities: $low (acceptable)" + echo "✅ Security scan PASSED: No concerning vulnerabilities found across all directories" + echo "Total Low vulnerabilities: $total_low (acceptable)" # Publish success metric aws cloudwatch put-metric-data \ From c83aabf27eef4b98a1103ce82663f1beb58af268 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 09:25:42 +0200 Subject: [PATCH 31/90] Correct metrics and functionality of the bash script --- scripts/security-scan.sh | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 49ad6627..410a56aa 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -13,21 +13,26 @@ run_security_scan() { echo "PR Branch (code being scanned): $head_ref" # Define directories to scan - local scan_dirs=("code-editor-src" "code-editor-src/remote" "code-editor-src/extensions" "code-editor-src/remote/web") + local scan_dirs=( + "code-editor-src" + "code-editor-src/remote" + "code-editor-src/extensions" + "code-editor-src/remote/web" + ) local scan_results=() # Scan each directory for dir in "${scan_dirs[@]}"; do echo "=== Scanning directory: $dir ===" - # Check if directory exists and has package.json + # Check if directory exists and has package-lock.json if [ ! -d "$dir" ]; then echo "Warning: Directory $dir does not exist, skipping..." continue fi - if [ ! -f "$dir/package.json" ]; then - echo "Warning: No package.json found in $dir, skipping..." + if [ ! -f "$dir/package-lock.json" ]; then + echo "Warning: No package-lock.json found in $dir, skipping..." continue fi @@ -61,8 +66,8 @@ run_security_scan() { echo "Publish success metric for Security Scan" aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ - --metric-name "ScanComplete" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning" \ + --metric-name "SecurityScanInvoked" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning,Target=$target" \ --value 1 } @@ -121,10 +126,6 @@ analyze_sbom_results() { if [ $dir_concerning -gt 0 ]; then has_failures=true echo "⚠️ Found $dir_concerning concerning vulnerabilities in $dir_name" - - # Display detailed vulnerability messages if available - echo "Vulnerability details for $dir_name:" - jq -r '.sbom.messages[]? | select(.vulnerability_message or .error_message) | " - \(.purl // "Unknown"): \(.vulnerability_message // .error_message // .info_message)"' "$result_file" || echo " No detailed messages available" else echo "✅ No concerning vulnerabilities in $dir_name" fi @@ -152,7 +153,7 @@ analyze_sbom_results() { --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ --dimensions "Repository=$repository,Workflow=SecurityScanning,Target=$target" \ - --value $total_concerning + --value 1 exit 1 else From cc0e5f7c55c5f41ca6fe296716a2ca1a3da16d6f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 09:37:53 +0200 Subject: [PATCH 32/90] Correct metrics and adjust file upload --- .github/workflows/security-scan.yml | 53 +++++++++++++++++++++-------- scripts/security-scan.sh | 6 ++-- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 5bdf3e8b..ed1c2413 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -40,6 +40,15 @@ jobs: echo "Base branch does not match allowed pattern - skipping security scan" echo "SHOULD_SCAN=false" >> $GITHUB_ENV fi + + - name: Assume IAM Role + if: env.SHOULD_SCAN == 'true' + id: assume-aws-iam-role + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} + aws-region: us-east-1 + role-session-name: publish-metrics - name: Checkout PR branch if: env.SHOULD_SCAN == 'true' @@ -80,39 +89,53 @@ jobs: run: | echo "Installing CycloneDX SBOM for npm" npm i -g @cyclonedx/cyclonedx-npm - - - name: Assume IAM Role - if: env.SHOULD_SCAN == 'true' - id: assume-aws-iam-role - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} - aws-region: us-east-1 - name: Run Security Scan if: env.SHOULD_SCAN == 'true' run: | ./scripts/security-scan.sh run-scan "${{ matrix.target }}" "${{ github.repository }}" "${{ github.head_ref }}" - - name: Output SBOM + - name: Upload SBOM Files + if: env.SHOULD_SCAN == 'true' uses: actions/upload-artifact@v4 with: - name: code-editor-sbom.json - path: code-editor-src/code-editor-sbom.json + name: sbom-files-${{ matrix.target }} + path: | + code-editor-src/*-sbom.json + code-editor-src/remote/*-sbom.json + code-editor-src/extensions/*-sbom.json + code-editor-src/remote/web/*-sbom.json retention-days: 90 - - name: Output Scan Results + - name: Upload Scan Result Files + if: env.SHOULD_SCAN == 'true' uses: actions/upload-artifact@v4 with: - name: sbom_scan_result.json - path: code-editor-src/sbom_scan_result.json + name: scan-results-${{ matrix.target }} + path: | + code-editor-src/*-scan-result.json + code-editor-src/remote/*-scan-result.json + code-editor-src/extensions/*-scan-result.json + code-editor-src/remote/web/*-scan-result.json retention-days: 90 - + - name: Analyze SBOM Scan Results if: env.SHOULD_SCAN == 'true' run: | ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" + - name: Publish Failure Metrics + if: failure() + run: | + echo "Job failed - publishing failure metrics" + + # Publish workflow failure metric + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanFailed" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning \ + --value 1 + get-branches-to-scan: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 410a56aa..3df2920b 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -67,7 +67,7 @@ run_security_scan() { aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning,Target=$target" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning" \ --value 1 } @@ -152,7 +152,7 @@ analyze_sbom_results() { aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning,Target=$target" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning" \ --value 1 exit 1 @@ -164,7 +164,7 @@ analyze_sbom_results() { aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanPassed" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning,Target=$target" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning" \ --value 1 fi } From 04781cf99a9c0a0e6ab2fbf91cdcc72d9548dc9c Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 10:28:34 +0200 Subject: [PATCH 33/90] Adjust metrics and directories --- .github/workflows/security-scan.yml | 2 +- scripts/security-scan.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ed1c2413..4b18a215 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -133,7 +133,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ --value 1 get-branches-to-scan: diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 3df2920b..33d2affc 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -38,7 +38,6 @@ run_security_scan() { # Generate SBOM for this directory echo "Generating SBOM for $dir" - cd "$dir" # Create a safe filename for the SBOM local safe_dir_name=$(echo "$dir" | sed 's/\//_/g') @@ -46,7 +45,7 @@ run_security_scan() { local result_file="${safe_dir_name}-scan-result.json" # 1.5 Spec Version compatible with Inspector's ScanSbom API - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" "$dir" echo "Invoking Inspector's ScanSbom API for $dir" aws inspector-scan scan-sbom --sbom "file://$sbom_file" > "$result_file" @@ -54,9 +53,6 @@ run_security_scan() { # Store the result file path for later analysis scan_results+=("$PWD/$result_file") - # Return to root directory - cd - > /dev/null - echo "Completed scan for $dir" done From a350935287f03fcce3aa64dee6dce8621e1dfba9 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 11:29:01 +0200 Subject: [PATCH 34/90] Fix scanning directories and commands --- .github/workflows/security-scan.yml | 6 --- scripts/security-scan.sh | 58 ++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 4b18a215..628bf24d 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -102,9 +102,6 @@ jobs: name: sbom-files-${{ matrix.target }} path: | code-editor-src/*-sbom.json - code-editor-src/remote/*-sbom.json - code-editor-src/extensions/*-sbom.json - code-editor-src/remote/web/*-sbom.json retention-days: 90 - name: Upload Scan Result Files @@ -114,9 +111,6 @@ jobs: name: scan-results-${{ matrix.target }} path: | code-editor-src/*-scan-result.json - code-editor-src/remote/*-scan-result.json - code-editor-src/extensions/*-scan-result.json - code-editor-src/remote/web/*-scan-result.json retention-days: 90 - name: Analyze SBOM Scan Results diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 33d2affc..2160122a 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -12,27 +12,39 @@ run_security_scan() { echo "Target: $target" echo "PR Branch (code being scanned): $head_ref" - # Define directories to scan - local scan_dirs=( - "code-editor-src" - "code-editor-src/remote" - "code-editor-src/extensions" - "code-editor-src/remote/web" + # Define directories to scan with their specific configurations + local scan_configs=( + "code-editor-src::root" + "remote::subdir" + "extensions::subdir" + "remote/web::subdir_ignore_errors" ) local scan_results=() # Scan each directory - for dir in "${scan_dirs[@]}"; do + for config in "${scan_configs[@]}"; do + local dir=$(echo "$config" | cut -d':' -f1) + local scan_type=$(echo "$config" | cut -d':' -f3) + echo "=== Scanning directory: $dir ===" + # For the first scan (code-editor-src), we need to check the root directory + # For others, we need to check subdirectories within code-editor-src + local check_dir + if [ "$scan_type" = "root" ]; then + check_dir="$dir" + else + check_dir="code-editor-src/$dir" + fi + # Check if directory exists and has package-lock.json - if [ ! -d "$dir" ]; then - echo "Warning: Directory $dir does not exist, skipping..." + if [ ! -d "$check_dir" ]; then + echo "Warning: Directory $check_dir does not exist, skipping..." continue fi - if [ ! -f "$dir/package-lock.json" ]; then - echo "Warning: No package-lock.json found in $dir, skipping..." + if [ ! -f "$check_dir/package-lock.json" ]; then + echo "Warning: No package-lock.json found in $check_dir, skipping..." continue fi @@ -44,8 +56,25 @@ run_security_scan() { local sbom_file="${safe_dir_name}-sbom.json" local result_file="${safe_dir_name}-scan-result.json" - # 1.5 Spec Version compatible with Inspector's ScanSbom API - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" "$dir" + # Handle different scan types + if [ "$scan_type" = "root" ]; then + # First scan: cd into code-editor-src and run scan there + echo "Scanning root directory: $dir" + cd "$dir" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" + + elif [ "$scan_type" = "subdir" ]; then + # Remaining scans: stay in code-editor-src and specify directory + echo "Scanning subdirectory: $dir from code-editor-src" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" "$dir" + + elif [ "$scan_type" = "subdir_ignore_errors" ]; then + # remote/web scan: add --ignore-npm-errors flag + # This is to ignore the extraneous error "npm error missing: tslib@*, required by @microsoft/applicationinsights-core-js@2.8.15" + # This behaviour is same for internal scanning. + echo "Scanning subdirectory: $dir from code-editor-src (ignoring npm errors)" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 --ignore-npm-errors -o "$sbom_file" "$dir" + fi echo "Invoking Inspector's ScanSbom API for $dir" aws inspector-scan scan-sbom --sbom "file://$sbom_file" > "$result_file" @@ -53,6 +82,9 @@ run_security_scan() { # Store the result file path for later analysis scan_results+=("$PWD/$result_file") + # Return to root directory for next iteration + cd - > /dev/null + echo "Completed scan for $dir" done From a9c596c8181e9b782914eb02671265d678ee61b8 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 11:38:58 +0200 Subject: [PATCH 35/90] Remove directory change --- scripts/security-scan.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 2160122a..71c869cd 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -82,9 +82,6 @@ run_security_scan() { # Store the result file path for later analysis scan_results+=("$PWD/$result_file") - # Return to root directory for next iteration - cd - > /dev/null - echo "Completed scan for $dir" done From 13a25cddae89f79cbfc03e3d72f0c526edbf0546 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 12:53:18 +0200 Subject: [PATCH 36/90] Add echo for pwd and dir --- scripts/security-scan.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 71c869cd..5c000eca 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -66,6 +66,8 @@ run_security_scan() { elif [ "$scan_type" = "subdir" ]; then # Remaining scans: stay in code-editor-src and specify directory echo "Scanning subdirectory: $dir from code-editor-src" + echo "pwd: $(pwd)" + echo "dir: $dir" cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" "$dir" elif [ "$scan_type" = "subdir_ignore_errors" ]; then From 9759ecc5240ec79febf6136f8a5d857a6ba2a8a2 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 13:12:49 +0200 Subject: [PATCH 37/90] Fix directories for security scanning --- scripts/security-scan.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 71c869cd..5fec9b32 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -66,6 +66,7 @@ run_security_scan() { elif [ "$scan_type" = "subdir" ]; then # Remaining scans: stay in code-editor-src and specify directory echo "Scanning subdirectory: $dir from code-editor-src" + cd code-editor-src cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" "$dir" elif [ "$scan_type" = "subdir_ignore_errors" ]; then @@ -73,6 +74,7 @@ run_security_scan() { # This is to ignore the extraneous error "npm error missing: tslib@*, required by @microsoft/applicationinsights-core-js@2.8.15" # This behaviour is same for internal scanning. echo "Scanning subdirectory: $dir from code-editor-src (ignoring npm errors)" + cd code-editor-src cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 --ignore-npm-errors -o "$sbom_file" "$dir" fi @@ -82,6 +84,9 @@ run_security_scan() { # Store the result file path for later analysis scan_results+=("$PWD/$result_file") + # Return to root directory for next iteration + cd - > /dev/null + echo "Completed scan for $dir" done From d530b8af9278944f1b4d7ed6b852a70d87335c25 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 13:13:14 +0200 Subject: [PATCH 38/90] Remove GitSecrets scan on push since it will run on merge --- .github/workflows/gitsecrets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gitsecrets.yml b/.github/workflows/gitsecrets.yml index 8e189762..452147e0 100644 --- a/.github/workflows/gitsecrets.yml +++ b/.github/workflows/gitsecrets.yml @@ -1,6 +1,6 @@ name: GitSecretsScan -on: [push, pull_request] +on: [pull_request] jobs: git-secret-check: From 971d4584f9b876efb59ab5123da708aaf0ef5146 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:00:02 +0200 Subject: [PATCH 39/90] Re-arrange the workflows to get all branches first and then run the scan --- .github/workflows/security-scan.yml | 223 ++++++++++++---------------- 1 file changed, 92 insertions(+), 131 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 628bf24d..bd26b3af 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -15,62 +15,106 @@ on: workflow_dispatch: jobs: - security-scan-pr: + get-branches-to-scan: runs-on: ubuntu-latest - if: github.event_name == 'pull_request' + outputs: + branches: ${{ steps.determine-branches.outputs.branches }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine branches to scan + id: determine-branches + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + # For PR events, validate base branch and use head ref if valid + base_ref="${{ github.base_ref }}" + echo "Base branch: $base_ref" + + if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then + echo "Base branch matches allowed pattern (main or digit.digit)" + echo "branches=[\"${{ github.head_ref }}\"]" >> $GITHUB_OUTPUT + echo "Branches to scan: [${{ github.head_ref }}]" + else + echo "Base branch does not match allowed pattern - no branches to scan" + echo "branches=[]" >> $GITHUB_OUTPUT + fi + else + # For scheduled/manual runs, get branches and filter by recent runs + echo "Getting branches for scheduled/manual run" + + # Get main branch and all version branches (*.*) + branches=$(git branch -r | grep -E 'origin/(main|[0-9]+\.[0-9]+)' | sed 's/origin\///' | tr '\n' ' ') + echo "Found branches: $branches" + + branches_to_scan="" + workflow_name="Security Scanning" + since_date=$(date -d '24 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) + + echo "Checking for workflow runs since: $since_date" + + # Check each branch for recent workflow runs + for branch in $branches; do + echo "Checking branch: $branch" + + # Get recent workflow runs for this specific workflow and branch + all_runs=$(gh run list --workflow="$workflow_name" --branch="$branch" --json startedAt,conclusion,headBranch --status success) + recent_runs=$(echo "$all_runs" | jq --arg since "$since_date" --arg branch "$branch" '.[] | select(.startedAt > $since and .headBranch == $branch)') + + if [ -n "$recent_runs" ]; then + echo "Skipping branch $branch - found recent workflow run in the last 24 hours" + else + echo "Adding branch $branch to scan list - no recent workflow runs" + branches_to_scan="$branches_to_scan $branch" + fi + done + + # Clean up extra spaces and convert to JSON array format + branches_to_scan=$(echo $branches_to_scan | xargs) + if [ -n "$branches_to_scan" ]; then + # Convert space-separated list to JSON array (compact format) + json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) + echo "branches=$json_branches" >> $GITHUB_OUTPUT + echo "Branches to scan: $json_branches" + else + echo "branches=[]" >> $GITHUB_OUTPUT + echo "No branches to scan - all have recent workflow runs" + fi + fi + + security-scan: + runs-on: ubuntu-latest + needs: [get-branches-to-scan] + if: needs.get-branches-to-scan.outputs.branches != '[]' environment: security-scanning-workflow-env permissions: id-token: write # Required for OIDC strategy: matrix: target: [code-editor-sagemaker-server] + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} steps: - - name: Validate base branch pattern - run: | - # PR trigger won't allow regex matching, it only allows *.*. - # So we add an additional validation check here. - - base_ref="${{ github.base_ref }}" - echo "Base branch: $base_ref" - - if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then - echo "Base branch matches allowed pattern (main or digit.digit)" - echo "SHOULD_SCAN=true" >> $GITHUB_ENV - else - echo "Base branch does not match allowed pattern - skipping security scan" - echo "SHOULD_SCAN=false" >> $GITHUB_ENV - fi - - - name: Assume IAM Role - if: env.SHOULD_SCAN == 'true' - id: assume-aws-iam-role - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} - aws-region: us-east-1 - role-session-name: publish-metrics - - - name: Checkout PR branch - if: env.SHOULD_SCAN == 'true' + - name: Checkout branch uses: actions/checkout@v4 with: - ref: ${{ github.head_ref }} + ref: ${{ matrix.branch }} submodules: recursive - name: Set up environment - if: env.SHOULD_SCAN == 'true' run: | echo "Installing required dependencies" sudo apt-get update sudo apt-get install -y quilt libkrb5-dev libx11-dev libxkbfile-dev libxml2-utils - name: Run patches script - if: env.SHOULD_SCAN == 'true' run: | - ./scripts/prepare-src.sh ${{ matrix.build-target }} + ./scripts/prepare-src.sh ${{ matrix.target }} - name: Set up Node.js - if: env.SHOULD_SCAN == 'true' uses: actions/setup-node@v4 with: node-version: '22' @@ -78,43 +122,45 @@ jobs: cache-dependency-path: 'code-editor-src/package-lock.json' - name: Install Code Editor Dependencies - if: env.SHOULD_SCAN == 'true' run: | cd code-editor-src echo "Installing Dependencies" npm ci - name: Install Security Scan Dependencies - if: env.SHOULD_SCAN == 'true' run: | echo "Installing CycloneDX SBOM for npm" npm i -g @cyclonedx/cyclonedx-npm + - name: Assume IAM Role + id: assume-aws-iam-role + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} + aws-region: us-east-1 + role-session-name: security-scan-${{ matrix.target }}-${{ matrix.branch }} + - name: Run Security Scan - if: env.SHOULD_SCAN == 'true' run: | - ./scripts/security-scan.sh run-scan "${{ matrix.target }}" "${{ github.repository }}" "${{ github.head_ref }}" - + ./scripts/security-scan.sh run-scan "${{ matrix.target }}" "${{ github.repository }}" "${{ matrix.branch }}" + - name: Upload SBOM Files - if: env.SHOULD_SCAN == 'true' uses: actions/upload-artifact@v4 with: - name: sbom-files-${{ matrix.target }} + name: sbom-files-${{ matrix.target }}-${{ matrix.branch }} path: | code-editor-src/*-sbom.json retention-days: 90 - name: Upload Scan Result Files - if: env.SHOULD_SCAN == 'true' uses: actions/upload-artifact@v4 with: - name: scan-results-${{ matrix.target }} + name: scan-results-${{ matrix.target }}-${{ matrix.branch }} path: | code-editor-src/*-scan-result.json retention-days: 90 - + - name: Analyze SBOM Scan Results - if: env.SHOULD_SCAN == 'true' run: | ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" @@ -128,89 +174,4 @@ jobs: --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ - --value 1 - - get-branches-to-scan: - runs-on: ubuntu-latest - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - outputs: - branches: ${{ steps.filter-branches.outputs.branches }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Get target branches - id: get-branches - run: | - # Get main branch and all version branches (*.*) - branches=$(git branch -r | grep -E 'origin/(main|[0-9]+\.[0-9]+)$' | sed 's/origin\///' | tr '\n' ' ') - echo "branches=$branches" >> $GITHUB_OUTPUT - echo "Found branches: $branches" - - - name: Filter branches based on recent workflow runs - id: filter-branches - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - branches="${{ steps.get-branches.outputs.branches }}" - branches_to_scan="" - workflow_name="Security Scanning" - since_date=$(date -d '24 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) - - echo "Checking for workflow runs since: $since_date" - - # Check each branch for recent workflow runs - for branch in $branches; do - echo "Checking branch: $branch" - - # Get recent workflow runs for this specific workflow and branch - all_runs=$(gh run list --workflow="$workflow_name" --branch="$branch" --json startedAt,conclusion,headBranch --status success) - recent_runs=$(echo "$all_runs" | jq --arg since "$since_date" --arg branch "$branch" '.[] | select(.startedAt > $since and .headBranch == $branch)') - - if [ -n "$recent_runs" ]; then - echo "Skipping branch $branch - found recent workflow run in the last 24 hours" - echo "Recent runs: $recent_runs" - else - echo "Adding branch $branch to scan list - no recent workflow runs" - branches_to_scan="$branches_to_scan $branch" - fi - done - - # Clean up extra spaces and convert to JSON array format - branches_to_scan=$(echo $branches_to_scan | xargs) - if [ -n "$branches_to_scan" ]; then - # Convert space-separated list to JSON array (compact format) - json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) - echo "branches=$json_branches" >> $GITHUB_OUTPUT - echo "Branches to scan: $json_branches" - else - echo "branches=[]" >> $GITHUB_OUTPUT - echo "No branches to scan - all have recent workflow runs" - fi - - security-scan-scheduled: - runs-on: ubuntu-latest - needs: [get-branches-to-scan] - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - strategy: - matrix: - branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ matrix.branch }} - - - name: Security Scanning Placeholder - run: | - echo "Security Scanning Started" - echo "Branch: ${{ matrix.branch }}" - echo "Trigger: ${{ github.event_name }}" - echo "Repository: ${{ github.repository }}" - echo "Timestamp: $(date -u)" - - # Placeholder for actual security scanning logic - # This is where you'll add your security scanning tools and commands - echo "TODO: Add security scanning implementation" \ No newline at end of file + --value 1 \ No newline at end of file From bf61149c89c4f34fa973f0732cd336884fd44723 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:01:54 +0200 Subject: [PATCH 40/90] Add comment for manual trigger --- .github/workflows/security-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index bd26b3af..485fbab5 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -12,6 +12,7 @@ on: schedule: - cron: '0 22 * * *' + # Trigger 3: Manual trigger workflow_dispatch: jobs: From 21ca6683601f00807a28afc5c8988199fe08d5d0 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:06:00 +0200 Subject: [PATCH 41/90] Change order for assuming AWS IAM Role --- .github/workflows/security-scan.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 485fbab5..2ab7695e 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -99,6 +99,14 @@ jobs: target: [code-editor-sagemaker-server] branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} steps: + - name: Assume IAM Role + id: assume-aws-iam-role + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} + aws-region: us-east-1 + role-session-name: security-scan-${{ matrix.target }}-${{ matrix.branch }} + - name: Checkout branch uses: actions/checkout@v4 with: @@ -133,14 +141,6 @@ jobs: echo "Installing CycloneDX SBOM for npm" npm i -g @cyclonedx/cyclonedx-npm - - name: Assume IAM Role - id: assume-aws-iam-role - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} - aws-region: us-east-1 - role-session-name: security-scan-${{ matrix.target }}-${{ matrix.branch }} - - name: Run Security Scan run: | ./scripts/security-scan.sh run-scan "${{ matrix.target }}" "${{ github.repository }}" "${{ matrix.branch }}" From 26d0d4ab26caed9e616d5e3b49961d27890e823e Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:22:05 +0200 Subject: [PATCH 42/90] Modify scan and adjust ignored errors directories --- scripts/security-scan.sh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 5fec9b32..0e231ee1 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -15,8 +15,8 @@ run_security_scan() { # Define directories to scan with their specific configurations local scan_configs=( "code-editor-src::root" - "remote::subdir" - "extensions::subdir" + "remote::subdir_ignore_errors" + "extensions::subdir_ignore_errors" "remote/web::subdir_ignore_errors" ) local scan_results=() @@ -63,19 +63,13 @@ run_security_scan() { cd "$dir" cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" - elif [ "$scan_type" = "subdir" ]; then - # Remaining scans: stay in code-editor-src and specify directory - echo "Scanning subdirectory: $dir from code-editor-src" - cd code-editor-src - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$sbom_file" "$dir" - elif [ "$scan_type" = "subdir_ignore_errors" ]; then - # remote/web scan: add --ignore-npm-errors flag - # This is to ignore the extraneous error "npm error missing: tslib@*, required by @microsoft/applicationinsights-core-js@2.8.15" + # Subdirectory scans with npm error handling: cd into directory and add --ignore-npm-errors flag + # This is to ignore extraneous npm errors that don't affect the security scan # This behaviour is same for internal scanning. - echo "Scanning subdirectory: $dir from code-editor-src (ignoring npm errors)" - cd code-editor-src - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 --ignore-npm-errors -o "$sbom_file" "$dir" + echo "Scanning subdirectory: $dir (ignoring npm errors)" + cd "$check_dir" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 --ignore-npm-errors -o "$sbom_file" fi echo "Invoking Inspector's ScanSbom API for $dir" From bd0e48d128f8d715f6f8664c541f80e08a4689e8 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:37:01 +0200 Subject: [PATCH 43/90] Without tar-fs --- .github/workflows/security-scan.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 441c9d69..2ab7695e 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -135,7 +135,6 @@ jobs: cd code-editor-src echo "Installing Dependencies" npm ci - npm install tar-fs@2.0.0 - name: Install Security Scan Dependencies run: | From 343f46c4cf7fd9c7b691b8b2670dcb8d650b8309 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:39:34 +0200 Subject: [PATCH 44/90] Add all targets to scan --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 2ab7695e..358b6b9c 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -96,7 +96,7 @@ jobs: id-token: write # Required for OIDC strategy: matrix: - target: [code-editor-sagemaker-server] + target: [code-editor-server, code-editor-sagemaker-server, code-editor-web-embedded, code-editor-web-embedded-with-terminal] branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} steps: - name: Assume IAM Role From a99b51d09decc456cd35d2a06d0ce9f6a9860d98 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:45:37 +0200 Subject: [PATCH 45/90] Update the scanned results files --- .github/workflows/security-scan.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 358b6b9c..f7814b70 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -151,6 +151,9 @@ jobs: name: sbom-files-${{ matrix.target }}-${{ matrix.branch }} path: | code-editor-src/*-sbom.json + code-editor-src/remote/*-sbom.json + code-editor-src/extensions/*-sbom.json + code-editor-src/remote/web/*-sbom.json retention-days: 90 - name: Upload Scan Result Files @@ -159,6 +162,9 @@ jobs: name: scan-results-${{ matrix.target }}-${{ matrix.branch }} path: | code-editor-src/*-scan-result.json + code-editor-src/remote/*-scan-result.json + code-editor-src/extensions/*-scan-result.json + code-editor-src/remote/web/*-scan-result.json retention-days: 90 - name: Analyze SBOM Scan Results From 0df86ddabe6ee803024da6d6e889ecb856fba2d8 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 14:48:19 +0200 Subject: [PATCH 46/90] Remove additional targets --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index f7814b70..0b3a1882 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -96,7 +96,7 @@ jobs: id-token: write # Required for OIDC strategy: matrix: - target: [code-editor-server, code-editor-sagemaker-server, code-editor-web-embedded, code-editor-web-embedded-with-terminal] + target: [code-editor-sagemaker-server] branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} steps: - name: Assume IAM Role From 504cf5914d24e48059931f41506142fa6e3d28e4 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 15:22:51 +0200 Subject: [PATCH 47/90] Pull script from main --- .github/workflows/security-scan.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 0b3a1882..45db81aa 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -112,6 +112,15 @@ jobs: with: ref: ${{ matrix.branch }} submodules: recursive + + - name: Update security scan script from main + run: | + # Older branches may not have the latest versions of the + # security scan scripts. So we pull the latest one from main + echo "Pulling latest security-scan.sh script from main branch" + git fetch origin main + git checkout origin/main -- scripts/security-scan.sh + echo "Updated security-scan.sh to latest version from main" - name: Set up environment run: | From 5259b4277f0601e2e5584614bc38085a6c24781a Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 15:31:01 +0200 Subject: [PATCH 48/90] Replace build targets invocation trigger --- .github/workflows/build-targets.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-targets.yaml b/.github/workflows/build-targets.yaml index e2cce736..bc5b3b54 100644 --- a/.github/workflows/build-targets.yaml +++ b/.github/workflows/build-targets.yaml @@ -1,6 +1,9 @@ name: Build Code Editor Targets on: - workflow_dispatch: + push: + branches: + - 'main' + - '*.*' jobs: build: From 723ad4f17689ed727353177bff0e5451e70498b4 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 15:31:47 +0200 Subject: [PATCH 49/90] Temporarily change trigger --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 45db81aa..35337215 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -10,7 +10,7 @@ on: # Trigger 2: Daily scheduled run at 22:00 UTC schedule: - - cron: '0 22 * * *' + - cron: '40 13 * * *' # Trigger 3: Manual trigger workflow_dispatch: From 882f4071bc0bbddbb61149c979ae821d86f6d9e2 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 15:46:19 +0200 Subject: [PATCH 50/90] Adjust trigger time and remove unused variable --- .github/workflows/security-scan.yml | 7 +++++-- scripts/security-scan.sh | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 35337215..1b7b31c4 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -8,9 +8,12 @@ on: - '*.*' types: [opened, edited, reopened, synchronize] - # Trigger 2: Daily scheduled run at 22:00 UTC + # Trigger 2: Daily scheduled run at 22:13 UTC + # Schedule it a random minute because most Github Actions are scheduled + # at the start of the hour and their invocation can get delayed. + # Ref: https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule schedule: - - cron: '40 13 * * *' + - cron: '13 22 * * *' # Trigger 3: Manual trigger workflow_dispatch: diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 0e231ee1..b2c1eeeb 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -112,7 +112,6 @@ analyze_sbom_results() { local total_medium=0 local total_other=0 local total_low=0 - local has_failures=false echo "=== SBOM Security Scan Results for $target ===" @@ -148,7 +147,6 @@ analyze_sbom_results() { # Check for concerning vulnerabilities in this directory local dir_concerning=$((critical + high + medium + other)) if [ $dir_concerning -gt 0 ]; then - has_failures=true echo "⚠️ Found $dir_concerning concerning vulnerabilities in $dir_name" else echo "✅ No concerning vulnerabilities in $dir_name" From a033ce5bf8bd124ee04cb5f1959011d2a2ba8e20 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 21:34:59 +0200 Subject: [PATCH 51/90] Add scanning functionality to scan Node JS binaries --- .github/workflows/security-scan.yml | 56 +++++++- scripts/generate-oss-attribution.sh | 10 +- scripts/security-scan.sh | 212 ++++++++++++++++++++++++---- 3 files changed, 243 insertions(+), 35 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 1b7b31c4..c0dad0a0 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -109,6 +109,14 @@ jobs: role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} aws-region: us-east-1 role-session-name: security-scan-${{ matrix.target }}-${{ matrix.branch }} + + - name: Publish Scan Invoked metric + run: | + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanInvoked" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning" \ + --value 1 - name: Checkout branch uses: actions/checkout@v4 @@ -153,9 +161,19 @@ jobs: echo "Installing CycloneDX SBOM for npm" npm i -g @cyclonedx/cyclonedx-npm + echo "Installing OSS Attribution Generator" + oss_attribution_version=$(cat build/oss-attribution-generator-version.txt) + npm i -g @electrovir/oss-attribution-generator@$oss_attribution_version + + echo "Installing Syft for SBOM generation" + curl -sSfL https://get.anchore.io/syft | sudo sh -s -- -b /usr/local/bin + echo "Syft installation completed" + syft version + + - name: Run Security Scan run: | - ./scripts/security-scan.sh run-scan "${{ matrix.target }}" "${{ github.repository }}" "${{ matrix.branch }}" + ./scripts/security-scan.sh scan-main-dependencies "${{ matrix.target }}" "${{ github.repository }}" "${{ matrix.branch }}" - name: Upload SBOM Files uses: actions/upload-artifact@v4 @@ -167,6 +185,7 @@ jobs: code-editor-src/extensions/*-sbom.json code-editor-src/remote/web/*-sbom.json retention-days: 90 + if-no-files-found: error - name: Upload Scan Result Files uses: actions/upload-artifact@v4 @@ -178,10 +197,43 @@ jobs: code-editor-src/extensions/*-scan-result.json code-editor-src/remote/web/*-scan-result.json retention-days: 90 + if-no-files-found: error - name: Analyze SBOM Scan Results run: | - ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" + ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" "scan_results_paths.txt" + + - name: Prepare Additional Node JS Dependencies for Scanning + run: | + ./scripts/security-scan.sh scan-additional-dependencies + + - name: Upload Additional Node.js SBOMs + uses: actions/upload-artifact@v4 + with: + name: additional-nodejs-sboms-${{ matrix.target }}-${{ matrix.branch }} + path: additional-node-js-sboms/ + retention-days: 90 + if-no-files-found: error + + - name: Upload Additional Inspector Scan Results + uses: actions/upload-artifact@v4 + with: + name: additional-inspector-results-${{ matrix.target }}-${{ matrix.branch }} + path: additional-scan-results/ + retention-days: 90 + if-no-files-found: error + + - name: Analyze Additional SBOM Scan Results + run: | + ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" "additional_scan_results_paths.txt" + + - name: Publish Scan Successful Metric + run: | + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanSuccessful" \ + --dimensions "Repository=$repository,Workflow=SecurityScanning" \ + --value 1 - name: Publish Failure Metrics if: failure() diff --git a/scripts/generate-oss-attribution.sh b/scripts/generate-oss-attribution.sh index 87207f00..81a31b2b 100755 --- a/scripts/generate-oss-attribution.sh +++ b/scripts/generate-oss-attribution.sh @@ -92,7 +92,10 @@ generate_oss_attribution() { check_unapproved_licenses "$target" "$BUILD_SRC_DIR" fi - npx --yes --package @electrovir/oss-attribution-generator@2.0.0 -- generate-attribution --baseDir "$BUILD_SRC_DIR" --outputDir "$oss_attribution_dir" + # Read OSS attribution generator version from file + local oss_attribution_version=$(cat "$ROOT_DIR/build/oss-attribution-generator-version.txt") + + npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_version -- generate-attribution --baseDir "$BUILD_SRC_DIR" --outputDir "$oss_attribution_dir" attribution_licenses=$(cat "$oss_attribution_dir/attribution.txt") read_status=0 @@ -167,7 +170,10 @@ generate_unified_oss_attribution() { echo "Generating unified OSS attribution for all targets" mkdir -p "$BUILD_DIR/private/oss-attribution" - npx --yes --package @electrovir/oss-attribution-generator@2.0.0 -- generate-attribution \ + # Read OSS attribution generator version from file + local oss_attribution_version=$(cat "$ROOT_DIR/build/oss-attribution-generator-version.txt") + + npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_version -- generate-attribution \ -b "${target_dirs[0]}" "${target_dirs[1]}" "${target_dirs[2]}" "${target_dirs[3]}" \ --outputDir "$BUILD_DIR/private/oss-attribution" diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index b2c1eeeb..37af463d 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -2,8 +2,8 @@ set -e -# Function to run the SBOM security scan -run_security_scan() { +# Function to scan main application dependencies +scan_main_dependencies() { local target="$1" local repository="$2" local head_ref="$3" @@ -86,23 +86,184 @@ run_security_scan() { # Store scan results paths in a file for the analyze step printf '%s\n' "${scan_results[@]}" > scan_results_paths.txt +} + +# Function to generate SBOMs for additional dependencies +generate_additional_sboms() { + echo "Generating SBOMs for additional dependencies" + + # Store current working directory + local root_dir=$(pwd) + + # Create directory for additional SBOMs + mkdir -p additional-node-js-sboms + + # 1. Generate SBOM for @electrovir/oss-attribution-generator + echo "Generating SBOM for @electrovir/oss-attribution-generator" + + # Find the global npm modules directory + global_npm_dir=$(npm list -g | head -1) + oss_attribution_dir="$global_npm_dir/node_modules/@electrovir/oss-attribution-generator" + + if [ -d "$oss_attribution_dir" ]; then + echo "Found OSS attribution generator at: $oss_attribution_dir" + cd "$oss_attribution_dir" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$root_dir/additional-node-js-sboms/oss-attribution-generator-sbom.json" + cd - > /dev/null + echo "Generated SBOM for OSS attribution generator" + else + echo "Error: OSS attribution generator not found at expected location: $oss_attribution_dir" + exit 1 + fi + + # 2. Generate SBOM for Node.js linux-x64 binary + echo "Generating SBOM for Node.js linux-x64 binary" + + # Read Node.js version from .npmrc file + if [ -f "code-editor-src/remote/.npmrc" ]; then + NODE_VERSION=$(grep 'target=' code-editor-src/remote/.npmrc | cut -d'"' -f2) + else + NODE_VERSION="22.15.1" # fallback version + fi + + node_x64_dir="nodejs-binaries/node-v$NODE_VERSION-linux-x64" + if [ -d "$node_x64_dir" ]; then + echo "Found Node.js x64 binary at: $node_x64_dir" + syft "$node_x64_dir" -o cyclonedx-json@1.5="$root_dir/additional-node-js-sboms/nodejs-x64-sbom.json" + echo "Generated SBOM for Node.js x64 binary" + else + echo "Error: Node.js x64 binary not found at expected location: $node_x64_dir" + exit 1 + fi + + # 3. Generate SBOM for Node.js linux-arm64 binary + echo "Generating SBOM for Node.js linux-arm64 binary" + + node_arm64_dir="nodejs-binaries/node-v$NODE_VERSION-linux-arm64" + if [ -d "$node_arm64_dir" ]; then + echo "Found Node.js ARM64 binary at: $node_arm64_dir" + syft "$node_arm64_dir" -o cyclonedx-json@1.5="$root_dir/additional-node-js-sboms/nodejs-arm64-sbom.json" + echo "Generated SBOM for Node.js ARM64 binary" + else + echo "Error: Node.js ARM64 binary not found at expected location: $node_arm64_dir" + exit 1 + fi + + # List generated SBOMs + echo "Generated additional SBOMs:" + ls -la additional-node-js-sboms/ + + echo "Additional SBOM generation completed successfully" +} + +# Function to scan additional SBOMs using AWS Inspector +scan_additional_sboms() { + echo "Scanning additional SBOMs with AWS Inspector" + + # First, download Node.js binaries + echo "Downloading Node.js binaries..." + download_nodejs_binaries + + # Then, generate additional SBOMs + echo "Generating additional SBOMs..." + generate_additional_sboms + + # Create directory for additional scan results + mkdir -p additional-scan-results + + # Check if additional SBOMs directory exists (should exist after generation) + if [ ! -d "additional-node-js-sboms" ]; then + echo "Error: additional-node-js-sboms directory not found after generation" + exit 1 + fi + + # Array to store scan result files for later analysis + local additional_scan_results=() + + # Scan each SBOM file in the additional-node-js-sboms directory + for sbom_file in additional-node-js-sboms/*.json; do + if [ ! -f "$sbom_file" ]; then + echo "Warning: No SBOM files found in additional-node-js-sboms directory" + continue + fi + + # Extract base filename without path and extension + local base_name=$(basename "$sbom_file" .json) + local result_file="additional-scan-results/${base_name}-scan-result.json" + + echo "Scanning SBOM: $sbom_file" + echo "Output will be saved to: $result_file" + + # Run AWS Inspector scan on the SBOM + aws inspector-scan scan-sbom --sbom "file://$sbom_file" > "$result_file" + + # Store the result file path for later analysis + additional_scan_results+=("$PWD/$result_file") + + echo "Completed scan for $base_name" + done + + # Store additional scan results paths in a file for the analyze step + printf '%s\n' "${additional_scan_results[@]}" > additional_scan_results_paths.txt + + echo "Additional SBOM scanning completed successfully" + echo "Scan results saved in additional-scan-results/ directory" + ls -la additional-scan-results/ +} - echo "Publish success metric for Security Scan" - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning" \ - --value 1 +# Function to download Node.js binaries for scanning +download_nodejs_binaries() { + echo "Downloading Node.js prebuilt binaries for scanning" + + # Create directory for Node.js binaries + mkdir -p nodejs-binaries + cd nodejs-binaries + + # Read Node.js version from .npmrc file + if [ -f "../code-editor-src/remote/.npmrc" ]; then + NODE_VERSION=$(grep 'target=' ../code-editor-src/remote/.npmrc | cut -d'"' -f2) + echo "Found Node.js version $NODE_VERSION in .npmrc" + else + NODE_VERSION="22.15.1" # fallback version + echo "Using fallback Node.js version $NODE_VERSION" + fi + + # Download Node.js binaries for both architectures + echo "Downloading Node.js v$NODE_VERSION for linux-x64" + curl -sSL "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" -o "node-v$NODE_VERSION-linux-x64.tar.xz" + + echo "Downloading Node.js v$NODE_VERSION for linux-arm64" + curl -sSL "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-arm64.tar.xz" -o "node-v$NODE_VERSION-linux-arm64.tar.xz" + + # Extract the downloaded files + echo "Extracting Node.js binaries" + tar -xf "node-v$NODE_VERSION-linux-x64.tar.xz" + tar -xf "node-v$NODE_VERSION-linux-arm64.tar.xz" + + echo "Node.js binaries downloaded and extracted:" + ls -la + + # Return to root directory + cd - > /dev/null + + echo "Node.js dependencies preparation completed" } # Function to analyze SBOM scan results analyze_sbom_results() { local target="$1" local repository="$2" + local results_file="$3" + + # Check if results file parameter is provided + if [ -z "$results_file" ]; then + echo "Error: Results file path is required as third parameter" + exit 1 + fi # Check if scan results paths file exists - if [ ! -f "scan_results_paths.txt" ]; then - echo "Error: Scan results paths file not found" + if [ ! -f "$results_file" ]; then + echo "Error: Scan results paths file '$results_file' not found" exit 1 fi @@ -152,7 +313,7 @@ analyze_sbom_results() { echo "✅ No concerning vulnerabilities in $dir_name" fi - done < scan_results_paths.txt + done < "$results_file" echo "" echo "=== TOTAL SCAN RESULTS ===" @@ -169,41 +330,30 @@ analyze_sbom_results() { if [ $total_concerning -gt 0 ]; then echo "❌ Security scan FAILED: Found $total_concerning concerning vulnerabilities across all directories" echo "Critical: $total_critical, High: $total_high, Medium: $total_medium, Other: $total_other" - - # Publish failure metric - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-name "SecurityScanFailed" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning" \ - --value 1 - exit 1 else echo "✅ Security scan PASSED: No concerning vulnerabilities found across all directories" echo "Total Low vulnerabilities: $total_low (acceptable)" - - # Publish success metric - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-name "SecurityScanPassed" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning" \ - --value 1 fi } # Main function to handle command line arguments main() { case "$1" in - "run-scan") - run_security_scan "$2" "$3" "$4" + "scan-main-dependencies") + scan_main_dependencies "$2" "$3" "$4" ;; "analyze-results") - analyze_sbom_results "$2" "$3" + analyze_sbom_results "$2" "$3" "$4" + ;; + "scan-additional-dependencies") + scan_additional_sboms ;; *) - echo "Usage: $0 {run-scan|analyze-results} [head_ref]" - echo " run-scan: Execute the SBOM security scan" + echo "Usage: $0 {scan-main-dependencies|analyze-results|scan-additional-dependencies}" + echo " scan-main-dependencies: Generate SBOMs and scan main application dependencies" echo " analyze-results: Analyze SBOM scan results and fail if vulnerabilities found" + echo " scan-additional-dependencies: Download, generate SBOMs, and scan additional Node.js dependencies" exit 1 ;; esac From ea49c7db6d32e4e7bc2aa588cb2dbb68d8a42553 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 21:43:54 +0200 Subject: [PATCH 52/90] Remove the repository variable --- .github/workflows/security-scan.yml | 10 +++++----- scripts/security-scan.sh | 12 +++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index c0dad0a0..c0c16ee9 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -115,7 +115,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ --value 1 - name: Checkout branch @@ -173,7 +173,7 @@ jobs: - name: Run Security Scan run: | - ./scripts/security-scan.sh scan-main-dependencies "${{ matrix.target }}" "${{ github.repository }}" "${{ matrix.branch }}" + ./scripts/security-scan.sh scan-main-dependencies "${{ matrix.target }}" "${{ matrix.branch }}" - name: Upload SBOM Files uses: actions/upload-artifact@v4 @@ -201,7 +201,7 @@ jobs: - name: Analyze SBOM Scan Results run: | - ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" "scan_results_paths.txt" + ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "scan_results_paths.txt" - name: Prepare Additional Node JS Dependencies for Scanning run: | @@ -225,14 +225,14 @@ jobs: - name: Analyze Additional SBOM Scan Results run: | - ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "${{ github.repository }}" "additional_scan_results_paths.txt" + ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "additional_scan_results_paths.txt" - name: Publish Scan Successful Metric run: | aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanSuccessful" \ - --dimensions "Repository=$repository,Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ --value 1 - name: Publish Failure Metrics diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 37af463d..cfa49c52 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -5,8 +5,7 @@ set -e # Function to scan main application dependencies scan_main_dependencies() { local target="$1" - local repository="$2" - local head_ref="$3" + local head_ref="$2" echo "Security Scanning Started" echo "Target: $target" @@ -252,12 +251,11 @@ download_nodejs_binaries() { # Function to analyze SBOM scan results analyze_sbom_results() { local target="$1" - local repository="$2" - local results_file="$3" + local results_file="$2" # Check if results file parameter is provided if [ -z "$results_file" ]; then - echo "Error: Results file path is required as third parameter" + echo "Error: Results file path is required as second parameter" exit 1 fi @@ -341,10 +339,10 @@ analyze_sbom_results() { main() { case "$1" in "scan-main-dependencies") - scan_main_dependencies "$2" "$3" "$4" + scan_main_dependencies "$2" "$3" ;; "analyze-results") - analyze_sbom_results "$2" "$3" "$4" + analyze_sbom_results "$2" "$3" ;; "scan-additional-dependencies") scan_additional_sboms From 1e7779787ae2e9a14e7274b540e20379db7949ac Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 21:59:11 +0200 Subject: [PATCH 53/90] Separate out a workflow for global dependency scanning --- .github/workflows/security-scan.yml | 81 +++++++++++++++---- .../oss-attribution-generator-version.txt | 1 + scripts/generate-oss-attribution.sh | 4 +- 3 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 config/pinned-versions/oss-attribution-generator-version.txt diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index c0c16ee9..88d7c82d 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -127,10 +127,9 @@ jobs: - name: Update security scan script from main run: | # Older branches may not have the latest versions of the - # security scan scripts. So we pull the latest one from main - echo "Pulling latest security-scan.sh script from main branch" - git fetch origin main - git checkout origin/main -- scripts/security-scan.sh + # security scan scripts. So we download the latest one from main + echo "Downloading latest security-scan.sh script from main branch" + curl -sSL "https://raw.githubusercontent.com/${{ github.repository }}/main/scripts/security-scan.sh" -o scripts/security-scan.sh echo "Updated security-scan.sh to latest version from main" - name: Set up environment @@ -162,7 +161,7 @@ jobs: npm i -g @cyclonedx/cyclonedx-npm echo "Installing OSS Attribution Generator" - oss_attribution_version=$(cat build/oss-attribution-generator-version.txt) + oss_attribution_version=$(cat config/pinned-versions/oss-attribution-generator-version.txt) npm i -g @electrovir/oss-attribution-generator@$oss_attribution_version echo "Installing Syft for SBOM generation" @@ -203,6 +202,68 @@ jobs: run: | ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "scan_results_paths.txt" + - name: Publish Scan Successful Metric + run: | + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanSuccessful" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --value 1 + + - name: Publish Failure Metrics + if: failure() + run: | + echo "Job failed - publishing failure metrics" + + # Publish workflow failure metric + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "SecurityScanFailed" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --value 1 + + security-scan-global-dependencies: + runs-on: ubuntu-latest + environment: security-scanning-workflow-env + permissions: + id-token: write # Required for OIDC + steps: + - name: Assume IAM Role + id: assume-aws-iam-role + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} + aws-region: us-east-1 + role-session-name: security-scan-global-dependencies + + - name: Checkout branch + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + submodules: recursive + + - name: Update security scan script from main + run: | + # Older branches may not have the latest versions of the + # security scan scripts. So we download the latest one from main + echo "Downloading latest security-scan.sh script from main branch" + curl -sSL "https://raw.githubusercontent.com/${{ github.repository }}/main/scripts/security-scan.sh" -o scripts/security-scan.sh + echo "Updated security-scan.sh to latest version from main" + + - name: Install Security Scan Dependencies + run: | + echo "Installing CycloneDX SBOM for npm" + npm i -g @cyclonedx/cyclonedx-npm + + echo "Installing OSS Attribution Generator" + oss_attribution_version=$(cat config/pinned-versions/oss-attribution-generator-version.txt) + npm i -g @electrovir/oss-attribution-generator@$oss_attribution_version + + echo "Installing Syft for SBOM generation" + curl -sSfL https://get.anchore.io/syft | sudo sh -s -- -b /usr/local/bin + echo "Syft installation completed" + syft version + - name: Prepare Additional Node JS Dependencies for Scanning run: | ./scripts/security-scan.sh scan-additional-dependencies @@ -226,15 +287,7 @@ jobs: - name: Analyze Additional SBOM Scan Results run: | ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "additional_scan_results_paths.txt" - - - name: Publish Scan Successful Metric - run: | - aws cloudwatch put-metric-data \ - --namespace "GitHub/Workflows" \ - --metric-name "SecurityScanSuccessful" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ - --value 1 - + - name: Publish Failure Metrics if: failure() run: | diff --git a/config/pinned-versions/oss-attribution-generator-version.txt b/config/pinned-versions/oss-attribution-generator-version.txt new file mode 100644 index 00000000..359a5b95 --- /dev/null +++ b/config/pinned-versions/oss-attribution-generator-version.txt @@ -0,0 +1 @@ +2.0.0 \ No newline at end of file diff --git a/scripts/generate-oss-attribution.sh b/scripts/generate-oss-attribution.sh index 81a31b2b..52c527be 100755 --- a/scripts/generate-oss-attribution.sh +++ b/scripts/generate-oss-attribution.sh @@ -93,7 +93,7 @@ generate_oss_attribution() { fi # Read OSS attribution generator version from file - local oss_attribution_version=$(cat "$ROOT_DIR/build/oss-attribution-generator-version.txt") + local oss_attribution_version=$(cat "$ROOT_DIR/config/pinned-versions/oss-attribution-generator-version.txt") npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_version -- generate-attribution --baseDir "$BUILD_SRC_DIR" --outputDir "$oss_attribution_dir" attribution_licenses=$(cat "$oss_attribution_dir/attribution.txt") @@ -171,7 +171,7 @@ generate_unified_oss_attribution() { mkdir -p "$BUILD_DIR/private/oss-attribution" # Read OSS attribution generator version from file - local oss_attribution_version=$(cat "$ROOT_DIR/build/oss-attribution-generator-version.txt") + local oss_attribution_version=$(cat "$ROOT_DIR/config/pinned-versions/oss-attribution-generator-version.txt") npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_version -- generate-attribution \ -b "${target_dirs[0]}" "${target_dirs[1]}" "${target_dirs[2]}" "${target_dirs[3]}" \ From 3ae5ae3fb57d3db326c5384dde6ea00151e18066 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 22:07:18 +0200 Subject: [PATCH 54/90] Correct third party src directory --- scripts/security-scan.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index cfa49c52..f4cb25a2 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -119,10 +119,11 @@ generate_additional_sboms() { echo "Generating SBOM for Node.js linux-x64 binary" # Read Node.js version from .npmrc file - if [ -f "code-editor-src/remote/.npmrc" ]; then + if [ -f "third-party-src/remote/.npmrc" ]; then NODE_VERSION=$(grep 'target=' code-editor-src/remote/.npmrc | cut -d'"' -f2) else - NODE_VERSION="22.15.1" # fallback version + echo "ERROR: No .npmrc file found" + exit 1 fi node_x64_dir="nodejs-binaries/node-v$NODE_VERSION-linux-x64" From 4754e5cc16215cb0739fe915755b6847fe1f8da5 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 22:13:19 +0200 Subject: [PATCH 55/90] Change matrix branch and remove auto trigger for build workflow --- .github/workflows/build-targets.yaml | 5 +---- .github/workflows/security-scan.yml | 6 +++--- scripts/security-scan.sh | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-targets.yaml b/.github/workflows/build-targets.yaml index bc5b3b54..e2cce736 100644 --- a/.github/workflows/build-targets.yaml +++ b/.github/workflows/build-targets.yaml @@ -1,9 +1,6 @@ name: Build Code Editor Targets on: - push: - branches: - - 'main' - - '*.*' + workflow_dispatch: jobs: build: diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 88d7c82d..ebafbf72 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -271,7 +271,7 @@ jobs: - name: Upload Additional Node.js SBOMs uses: actions/upload-artifact@v4 with: - name: additional-nodejs-sboms-${{ matrix.target }}-${{ matrix.branch }} + name: additional-nodejs-sboms-${{ github.head_ref }} path: additional-node-js-sboms/ retention-days: 90 if-no-files-found: error @@ -279,14 +279,14 @@ jobs: - name: Upload Additional Inspector Scan Results uses: actions/upload-artifact@v4 with: - name: additional-inspector-results-${{ matrix.target }}-${{ matrix.branch }} + name: additional-inspector-results-${{ github.head_ref }} path: additional-scan-results/ retention-days: 90 if-no-files-found: error - name: Analyze Additional SBOM Scan Results run: | - ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "additional_scan_results_paths.txt" + ./scripts/security-scan.sh analyze-results "Global Dependencies" "additional_scan_results_paths.txt" - name: Publish Failure Metrics if: failure() diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index f4cb25a2..8011b486 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -120,7 +120,7 @@ generate_additional_sboms() { # Read Node.js version from .npmrc file if [ -f "third-party-src/remote/.npmrc" ]; then - NODE_VERSION=$(grep 'target=' code-editor-src/remote/.npmrc | cut -d'"' -f2) + NODE_VERSION=$(grep 'target=' third-party-src/remote/.npmrc | cut -d'"' -f2) else echo "ERROR: No .npmrc file found" exit 1 From ed8b69c6b837934d57cd35fe316a41c787a47ea6 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 23:18:11 +0200 Subject: [PATCH 56/90] Add GitHub scanning for Microsoft VS Code --- .github/workflows/security-scan.yml | 18 +- config/pinned-versions/semver-version.txt | 1 + scripts/security-scan.sh | 192 ++++++++++++++++++---- 3 files changed, 173 insertions(+), 38 deletions(-) create mode 100644 config/pinned-versions/semver-version.txt diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ebafbf72..64e7f7ca 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -224,9 +224,13 @@ jobs: security-scan-global-dependencies: runs-on: ubuntu-latest + needs: [get-branches-to-scan] environment: security-scanning-workflow-env permissions: id-token: write # Required for OIDC + strategy: + matrix: + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} steps: - name: Assume IAM Role id: assume-aws-iam-role @@ -239,7 +243,7 @@ jobs: - name: Checkout branch uses: actions/checkout@v4 with: - ref: ${{ github.head_ref }} + ref: ${{ matrix.branch }} submodules: recursive - name: Update security scan script from main @@ -258,6 +262,10 @@ jobs: echo "Installing OSS Attribution Generator" oss_attribution_version=$(cat config/pinned-versions/oss-attribution-generator-version.txt) npm i -g @electrovir/oss-attribution-generator@$oss_attribution_version + + echo "Installing semver" + semver_version=$(cat config/pinned-versions/semver-version.txt) + npm i -g semver@$semver_version echo "Installing Syft for SBOM generation" curl -sSfL https://get.anchore.io/syft | sudo sh -s -- -b /usr/local/bin @@ -271,7 +279,7 @@ jobs: - name: Upload Additional Node.js SBOMs uses: actions/upload-artifact@v4 with: - name: additional-nodejs-sboms-${{ github.head_ref }} + name: additional-nodejs-sboms-${{ matrix.branch }} path: additional-node-js-sboms/ retention-days: 90 if-no-files-found: error @@ -279,7 +287,7 @@ jobs: - name: Upload Additional Inspector Scan Results uses: actions/upload-artifact@v4 with: - name: additional-inspector-results-${{ github.head_ref }} + name: additional-inspector-results-${{ matrix.branch }} path: additional-scan-results/ retention-days: 90 if-no-files-found: error @@ -287,6 +295,10 @@ jobs: - name: Analyze Additional SBOM Scan Results run: | ./scripts/security-scan.sh analyze-results "Global Dependencies" "additional_scan_results_paths.txt" + + - name: Scan GitHub Security Advisories + run: | + ./scripts/security-scan.sh scan-github-advisories - name: Publish Failure Metrics if: failure() diff --git a/config/pinned-versions/semver-version.txt b/config/pinned-versions/semver-version.txt new file mode 100644 index 00000000..94204632 --- /dev/null +++ b/config/pinned-versions/semver-version.txt @@ -0,0 +1 @@ +7.7.2 \ No newline at end of file diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 8011b486..bb0f0ef5 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -104,50 +104,41 @@ generate_additional_sboms() { global_npm_dir=$(npm list -g | head -1) oss_attribution_dir="$global_npm_dir/node_modules/@electrovir/oss-attribution-generator" - if [ -d "$oss_attribution_dir" ]; then - echo "Found OSS attribution generator at: $oss_attribution_dir" - cd "$oss_attribution_dir" - cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$root_dir/additional-node-js-sboms/oss-attribution-generator-sbom.json" - cd - > /dev/null - echo "Generated SBOM for OSS attribution generator" - else - echo "Error: OSS attribution generator not found at expected location: $oss_attribution_dir" - exit 1 - fi + echo "Found OSS attribution generator at: $oss_attribution_dir" + cd "$oss_attribution_dir" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$root_dir/additional-node-js-sboms/oss-attribution-generator-sbom.json" + cd - > /dev/null + echo "Generated SBOM for OSS attribution generator" + + # 2. Generate SBOM for semver package + echo "Generating SBOM for semver package" - # 2. Generate SBOM for Node.js linux-x64 binary + semver_dir="$global_npm_dir/node_modules/semver" + + echo "Found semver package at: $semver_dir" + cd "$semver_dir" + cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$root_dir/additional-node-js-sboms/semver-sbom.json" + cd - > /dev/null + echo "Generated SBOM for semver package" + + # 3. Generate SBOM for Node.js linux-x64 binary echo "Generating SBOM for Node.js linux-x64 binary" # Read Node.js version from .npmrc file - if [ -f "third-party-src/remote/.npmrc" ]; then - NODE_VERSION=$(grep 'target=' third-party-src/remote/.npmrc | cut -d'"' -f2) - else - echo "ERROR: No .npmrc file found" - exit 1 - fi + NODE_VERSION=$(grep 'target=' third-party-src/remote/.npmrc | cut -d'"' -f2) node_x64_dir="nodejs-binaries/node-v$NODE_VERSION-linux-x64" - if [ -d "$node_x64_dir" ]; then - echo "Found Node.js x64 binary at: $node_x64_dir" - syft "$node_x64_dir" -o cyclonedx-json@1.5="$root_dir/additional-node-js-sboms/nodejs-x64-sbom.json" - echo "Generated SBOM for Node.js x64 binary" - else - echo "Error: Node.js x64 binary not found at expected location: $node_x64_dir" - exit 1 - fi + echo "Found Node.js x64 binary at: $node_x64_dir" + syft "$node_x64_dir" -o cyclonedx-json@1.5="$root_dir/additional-node-js-sboms/nodejs-x64-sbom.json" + echo "Generated SBOM for Node.js x64 binary" - # 3. Generate SBOM for Node.js linux-arm64 binary + # 4. Generate SBOM for Node.js linux-arm64 binary echo "Generating SBOM for Node.js linux-arm64 binary" node_arm64_dir="nodejs-binaries/node-v$NODE_VERSION-linux-arm64" - if [ -d "$node_arm64_dir" ]; then - echo "Found Node.js ARM64 binary at: $node_arm64_dir" - syft "$node_arm64_dir" -o cyclonedx-json@1.5="$root_dir/additional-node-js-sboms/nodejs-arm64-sbom.json" - echo "Generated SBOM for Node.js ARM64 binary" - else - echo "Error: Node.js ARM64 binary not found at expected location: $node_arm64_dir" - exit 1 - fi + echo "Found Node.js ARM64 binary at: $node_arm64_dir" + syft "$node_arm64_dir" -o cyclonedx-json@1.5="$root_dir/additional-node-js-sboms/nodejs-arm64-sbom.json" + echo "Generated SBOM for Node.js ARM64 binary" # List generated SBOMs echo "Generated additional SBOMs:" @@ -336,6 +327,133 @@ analyze_sbom_results() { fi } +# Function to scan GitHub security advisories for microsoft/vscode +scan_github_advisories() { + echo "Scanning GitHub security advisories for microsoft/vscode" + + local repo_owner="microsoft" + local repo_name="vscode" + local vscode_version=$(jq -r '.version' third-party-src/package.json) + + echo "Found VS Code version: $vscode_version" + + echo "Fetching security advisories from GitHub API for $repo_owner/$repo_name" + + # Fetch security advisories using GitHub CLI + local temp_file=$(mktemp) + + # Make API request using gh cli with proper headers + if ! gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/$repo_owner/$repo_name/security-advisories" > "$temp_file"; then + echo "Error: Failed to fetch GitHub security advisories using GitHub CLI" + echo "Make sure GitHub CLI is installed and authenticated" + rm -f "$temp_file" + exit 1 + fi + + # Check if the response is valid JSON and not an error + if ! jq empty "$temp_file" 2>/dev/null; then + echo "Error: Invalid JSON response from GitHub API" + cat "$temp_file" + rm -f "$temp_file" + exit 1 + fi + + # Count total advisories + local total_advisories=$(jq length "$temp_file") + echo "Found $total_advisories total advisories for $repo_owner/$repo_name" + + if [ "$total_advisories" -eq 0 ]; then + echo "✅ No security advisories found for microsoft/vscode" + rm -f "$temp_file" + return 0 + fi + + # Process advisories + local concerning_advisories=0 + local advisory_results=$(mktemp) + + echo "" + echo "=== GITHUB SECURITY ADVISORIES ANALYSIS ===" + + # Extract and analyze each advisory + jq -c '.[]' "$temp_file" | while read -r advisory; do + local ghsa_id=$(echo "$advisory" | jq -r '.ghsa_id // "N/A"') + local cve_id=$(echo "$advisory" | jq -r '.cve_id // "N/A"') + local severity=$(echo "$advisory" | jq -r '.severity // "unknown"') + local summary=$(echo "$advisory" | jq -r '.summary // "No summary available"') + local published_at=$(echo "$advisory" | jq -r '.published_at // "N/A"') + + echo "" + echo "Advisory: $ghsa_id" + [ "$cve_id" != "N/A" ] && echo "CVE: $cve_id" + echo "Severity: $severity" + echo "Published: $published_at" + echo "Summary: $summary" + + # Check if current version is affected using semver + # Extract all vulnerable version ranges for this advisory + echo "$advisory" | jq -r '.vulnerabilities[].vulnerable_version_range // empty' | while read -r vulnerable_range; do + if [ -n "$vulnerable_range" ]; then + echo "Vulnerable versions: $vulnerable_range" + + # Use semver satisfies to check if current version satisfies the vulnerable range + if semver satisfies "$vscode_version" "$vulnerable_range" >/dev/null 2>&1; then + echo "⚠️ Version $vscode_version is affected by this advisory (satisfies range: $vulnerable_range)" + echo "1" > /tmp/is_affected_$$ + else + echo "✅ Version $vscode_version does not satisfy vulnerable range: $vulnerable_range" + fi + fi + done + + # Check if any range matched (since the while loop runs in a subshell) + if [ -f "/tmp/is_affected_$$" ]; then + concerning_advisories=$((concerning_advisories + 1)) + rm -f "/tmp/is_affected_$$" + fi + + # If no vulnerable ranges specified, assume potentially affected + local has_ranges=$(echo "$advisory" | jq -r '.vulnerabilities[].vulnerable_version_range // empty' | wc -l) + if [ "$has_ranges" -eq 0 ]; then + echo "⚠️ No version range specified - assuming potentially affected" + concerning_advisories=$((concerning_advisories + 1)) + fi + + # Count moderate/high/critical severity advisories as concerning + if [ "$severity" = "moderate" ] || [ "$severity" = "high" ] || [ "$severity" = "critical" ]; then + concerning_advisories=$((concerning_advisories + 1)) + fi + done + + # Store the concerning count for the parent process + echo "$concerning_advisories" > "$advisory_results" + + # Read the result back (since the while loop runs in a subshell) + concerning_advisories=$(cat "$advisory_results") + + echo "" + echo "=== GITHUB ADVISORIES SUMMARY ===" + echo "Total advisories found: $total_advisories" + echo "Concerning advisories: $concerning_advisories" + echo "==================================================" + + # Clean up temp files + rm -f "$temp_file" "$advisory_results" + + # Determine if we should fail based on concerning advisories + if [ "$concerning_advisories" -gt 0 ]; then + echo "⚠️ Found $concerning_advisories concerning GitHub security advisories for microsoft/vscode" + echo "Review the advisories above to determine if they affect your VS Code integration" + exit 1 + else + echo "✅ No concerning GitHub security advisories found for microsoft/vscode" + return 0 + fi +} + # Main function to handle command line arguments main() { case "$1" in @@ -348,11 +466,15 @@ main() { "scan-additional-dependencies") scan_additional_sboms ;; + "scan-github-advisories") + scan_github_advisories + ;; *) - echo "Usage: $0 {scan-main-dependencies|analyze-results|scan-additional-dependencies}" + echo "Usage: $0 {scan-main-dependencies|analyze-results|scan-additional-dependencies|scan-github-advisories}" echo " scan-main-dependencies: Generate SBOMs and scan main application dependencies" echo " analyze-results: Analyze SBOM scan results and fail if vulnerabilities found" echo " scan-additional-dependencies: Download, generate SBOMs, and scan additional Node.js dependencies" + echo " scan-github-advisories: Scan GitHub security advisories for microsoft/vscode" exit 1 ;; esac From a4800dfc2ce4267959ef8799b9b1d1be9c33c413 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 23:25:07 +0200 Subject: [PATCH 57/90] Add semver installation step --- scripts/security-scan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index bb0f0ef5..ebaedf64 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -117,6 +117,7 @@ generate_additional_sboms() { echo "Found semver package at: $semver_dir" cd "$semver_dir" + npm install cyclonedx-npm --omit dev --output-reproducible --spec-version 1.5 -o "$root_dir/additional-node-js-sboms/semver-sbom.json" cd - > /dev/null echo "Generated SBOM for semver package" From 7c5c018b621c2b7a4c97e0d3e685021d246b711f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 23:28:24 +0200 Subject: [PATCH 58/90] Add token for using GitHub CLI --- .github/workflows/security-scan.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 64e7f7ca..9e74e857 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -297,6 +297,8 @@ jobs: ./scripts/security-scan.sh analyze-results "Global Dependencies" "additional_scan_results_paths.txt" - name: Scan GitHub Security Advisories + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ./scripts/security-scan.sh scan-github-advisories From 01fd0ef6304d6cba3e9be5c79f210bb00b1b1779 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 23:37:54 +0200 Subject: [PATCH 59/90] Test intentional failure for GitHub scan --- .github/workflows/security-scan.yml | 9 --------- scripts/security-scan.sh | 3 ++- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 9e74e857..f5cdda74 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -160,15 +160,6 @@ jobs: echo "Installing CycloneDX SBOM for npm" npm i -g @cyclonedx/cyclonedx-npm - echo "Installing OSS Attribution Generator" - oss_attribution_version=$(cat config/pinned-versions/oss-attribution-generator-version.txt) - npm i -g @electrovir/oss-attribution-generator@$oss_attribution_version - - echo "Installing Syft for SBOM generation" - curl -sSfL https://get.anchore.io/syft | sudo sh -s -- -b /usr/local/bin - echo "Syft installation completed" - syft version - - name: Run Security Scan run: | diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index ebaedf64..2f95d66c 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -334,7 +334,8 @@ scan_github_advisories() { local repo_owner="microsoft" local repo_name="vscode" - local vscode_version=$(jq -r '.version' third-party-src/package.json) + # local vscode_version=$(jq -r '.version' third-party-src/package.json) + local vscode_version="1.95.0" echo "Found VS Code version: $vscode_version" From 6201aacee00ad369a3d90b72b6a5251252bde14f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 23:45:24 +0200 Subject: [PATCH 60/90] Use semver range --- scripts/security-scan.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 2f95d66c..4a7a4bdd 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -401,12 +401,12 @@ scan_github_advisories() { if [ -n "$vulnerable_range" ]; then echo "Vulnerable versions: $vulnerable_range" - # Use semver satisfies to check if current version satisfies the vulnerable range - if semver satisfies "$vscode_version" "$vulnerable_range" >/dev/null 2>&1; then - echo "⚠️ Version $vscode_version is affected by this advisory (satisfies range: $vulnerable_range)" + # Use semver range to check if current version is in the vulnerable range + if semver range "$vulnerable_range" "$vscode_version" >/dev/null 2>&1; then + echo "⚠️ Version $vscode_version is affected by this advisory (in range: $vulnerable_range)" echo "1" > /tmp/is_affected_$$ else - echo "✅ Version $vscode_version does not satisfy vulnerable range: $vulnerable_range" + echo "✅ Version $vscode_version is not in vulnerable range: $vulnerable_range" fi fi done From 873c2035548e6586c442ae082e4931a010c4edb1 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Tue, 26 Aug 2025 23:46:56 +0200 Subject: [PATCH 61/90] Use semver --range --- scripts/security-scan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 4a7a4bdd..f8415e6d 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -402,7 +402,7 @@ scan_github_advisories() { echo "Vulnerable versions: $vulnerable_range" # Use semver range to check if current version is in the vulnerable range - if semver range "$vulnerable_range" "$vscode_version" >/dev/null 2>&1; then + if semver --range "$vulnerable_range" "$vscode_version" >/dev/null 2>&1; then echo "⚠️ Version $vscode_version is affected by this advisory (in range: $vulnerable_range)" echo "1" > /tmp/is_affected_$$ else From ce4929223c5a26e3fafaa922f778c39437901daf Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 00:01:14 +0200 Subject: [PATCH 62/90] Update Github Scan Logic --- scripts/security-scan.sh | 54 +++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index f8415e6d..729818d6 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -375,13 +375,18 @@ scan_github_advisories() { # Process advisories local concerning_advisories=0 - local advisory_results=$(mktemp) + local advisory_count_file=$(mktemp) + echo "0" > "$advisory_count_file" echo "" echo "=== GITHUB SECURITY ADVISORIES ANALYSIS ===" - # Extract and analyze each advisory - jq -c '.[]' "$temp_file" | while read -r advisory; do + # Process each advisory + local advisory_index=0 + local total_advisories_count=$(jq length "$temp_file") + + while [ $advisory_index -lt $total_advisories_count ]; do + local advisory=$(jq -c ".[$advisory_index]" "$temp_file") local ghsa_id=$(echo "$advisory" | jq -r '.ghsa_id // "N/A"') local cve_id=$(echo "$advisory" | jq -r '.cve_id // "N/A"') local severity=$(echo "$advisory" | jq -r '.severity // "unknown"') @@ -395,47 +400,40 @@ scan_github_advisories() { echo "Published: $published_at" echo "Summary: $summary" + local is_version_affected=false + # Check if current version is affected using semver - # Extract all vulnerable version ranges for this advisory - echo "$advisory" | jq -r '.vulnerabilities[].vulnerable_version_range // empty' | while read -r vulnerable_range; do - if [ -n "$vulnerable_range" ]; then + local vulnerable_ranges=$(echo "$advisory" | jq -r '.vulnerabilities[].vulnerable_version_range // empty') + + if [ -n "$vulnerable_ranges" ]; then + echo "$vulnerable_ranges" | while read -r vulnerable_range; do echo "Vulnerable versions: $vulnerable_range" # Use semver range to check if current version is in the vulnerable range if semver --range "$vulnerable_range" "$vscode_version" >/dev/null 2>&1; then echo "⚠️ Version $vscode_version is affected by this advisory (in range: $vulnerable_range)" - echo "1" > /tmp/is_affected_$$ + is_version_affected=true else echo "✅ Version $vscode_version is not in vulnerable range: $vulnerable_range" fi - fi - done - - # Check if any range matched (since the while loop runs in a subshell) - if [ -f "/tmp/is_affected_$$" ]; then - concerning_advisories=$((concerning_advisories + 1)) - rm -f "/tmp/is_affected_$$" - fi - - # If no vulnerable ranges specified, assume potentially affected - local has_ranges=$(echo "$advisory" | jq -r '.vulnerabilities[].vulnerable_version_range // empty' | wc -l) - if [ "$has_ranges" -eq 0 ]; then + done + + + else echo "⚠️ No version range specified - assuming potentially affected" - concerning_advisories=$((concerning_advisories + 1)) + is_version_affected=true fi - # Count moderate/high/critical severity advisories as concerning - if [ "$severity" = "moderate" ] || [ "$severity" = "high" ] || [ "$severity" = "critical" ]; then + # Count concerning advisories based on combined criteria + # Advisory is concerning if BOTH conditions are met: + # 1. Version is affected AND 2. Severity is moderate/high/critical + if [ "$is_version_affected" = true ] && ([ "$severity" = "moderate" ] || [ "$severity" = "high" ] || [ "$severity" = "critical" ]); then concerning_advisories=$((concerning_advisories + 1)) fi + + advisory_index=$((advisory_index + 1)) done - # Store the concerning count for the parent process - echo "$concerning_advisories" > "$advisory_results" - - # Read the result back (since the while loop runs in a subshell) - concerning_advisories=$(cat "$advisory_results") - echo "" echo "=== GITHUB ADVISORIES SUMMARY ===" echo "Total advisories found: $total_advisories" From e7ddcd0859187ce5d5136da80e128c4e0d1bff9d Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 00:06:41 +0200 Subject: [PATCH 63/90] Use for loop for vulnerable versions --- scripts/security-scan.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 729818d6..5f8df61b 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -406,7 +406,18 @@ scan_github_advisories() { local vulnerable_ranges=$(echo "$advisory" | jq -r '.vulnerabilities[].vulnerable_version_range // empty') if [ -n "$vulnerable_ranges" ]; then - echo "$vulnerable_ranges" | while read -r vulnerable_range; do + # Process each vulnerable range + local ranges_array=() + + # Convert vulnerable ranges to array + while IFS= read -r range; do + if [ -n "$range" ]; then + ranges_array+=("$range") + fi + done <<< "$vulnerable_ranges" + + # Check each range + for vulnerable_range in "${ranges_array[@]}"; do echo "Vulnerable versions: $vulnerable_range" # Use semver range to check if current version is in the vulnerable range From eb6545257197d2a7cc47134945a37ad93a9329af Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 00:11:32 +0200 Subject: [PATCH 64/90] Remove unused files --- scripts/security-scan.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 5f8df61b..01e823e1 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -375,8 +375,6 @@ scan_github_advisories() { # Process advisories local concerning_advisories=0 - local advisory_count_file=$(mktemp) - echo "0" > "$advisory_count_file" echo "" echo "=== GITHUB SECURITY ADVISORIES ANALYSIS ===" @@ -452,7 +450,7 @@ scan_github_advisories() { echo "==================================================" # Clean up temp files - rm -f "$temp_file" "$advisory_results" + rm -f "$temp_file" # Determine if we should fail based on concerning advisories if [ "$concerning_advisories" -gt 0 ]; then From 9569715bc1a86202f0660a1b23cbceadbcaa516c Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 00:18:00 +0200 Subject: [PATCH 65/90] Add echo for incrementing count --- scripts/security-scan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 01e823e1..a1aab73d 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -437,6 +437,7 @@ scan_github_advisories() { # Advisory is concerning if BOTH conditions are met: # 1. Version is affected AND 2. Severity is moderate/high/critical if [ "$is_version_affected" = true ] && ([ "$severity" = "moderate" ] || [ "$severity" = "high" ] || [ "$severity" = "critical" ]); then + echo "Incrementing count" concerning_advisories=$((concerning_advisories + 1)) fi From 22b96d8f05c5e41b1036fb5300d6208031c2f564 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 00:24:22 +0200 Subject: [PATCH 66/90] Adjust severity to medium --- scripts/security-scan.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index a1aab73d..adf8a784 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -435,8 +435,8 @@ scan_github_advisories() { # Count concerning advisories based on combined criteria # Advisory is concerning if BOTH conditions are met: - # 1. Version is affected AND 2. Severity is moderate/high/critical - if [ "$is_version_affected" = true ] && ([ "$severity" = "moderate" ] || [ "$severity" = "high" ] || [ "$severity" = "critical" ]); then + # 1. Version is affected AND 2. Severity is medium/high/critical + if [ "$is_version_affected" = true ] && ([ "$severity" = "medium" ] || [ "$severity" = "high" ] || [ "$severity" = "critical" ]); then echo "Incrementing count" concerning_advisories=$((concerning_advisories + 1)) fi From f52d95ea5ad85cc4e3d51eb3f5cca33320e242f5 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 00:26:38 +0200 Subject: [PATCH 67/90] Fix version --- scripts/security-scan.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index adf8a784..403fbd73 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -334,8 +334,7 @@ scan_github_advisories() { local repo_owner="microsoft" local repo_name="vscode" - # local vscode_version=$(jq -r '.version' third-party-src/package.json) - local vscode_version="1.95.0" + local vscode_version=$(jq -r '.version' third-party-src/package.json) echo "Found VS Code version: $vscode_version" From d901ae7dc9d7593ac54712cae58bc5966a321673 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 00:51:13 +0200 Subject: [PATCH 68/90] Fix Node version detection --- scripts/security-scan.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 403fbd73..253dc19b 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -212,12 +212,12 @@ download_nodejs_binaries() { cd nodejs-binaries # Read Node.js version from .npmrc file - if [ -f "../code-editor-src/remote/.npmrc" ]; then - NODE_VERSION=$(grep 'target=' ../code-editor-src/remote/.npmrc | cut -d'"' -f2) + if [ -f "../third-party-src/remote/.npmrc" ]; then + NODE_VERSION=$(grep 'target=' ../third-party-src/remote/.npmrc | cut -d'"' -f2) echo "Found Node.js version $NODE_VERSION in .npmrc" else - NODE_VERSION="22.15.1" # fallback version - echo "Using fallback Node.js version $NODE_VERSION" + echo "ERROR: Unable to determine NODE_VERSION" + exit 1 fi # Download Node.js binaries for both architectures From c3785e2b3d7a082a9b082184a21cef5e85a5bf7b Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Wed, 27 Aug 2025 15:51:25 +0200 Subject: [PATCH 69/90] Sync with published PR --- .github/workflows/gitsecrets.yml | 7 ++++++- .github/workflows/security-scan.yml | 8 ++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gitsecrets.yml b/.github/workflows/gitsecrets.yml index 452147e0..411a3a5c 100644 --- a/.github/workflows/gitsecrets.yml +++ b/.github/workflows/gitsecrets.yml @@ -1,6 +1,11 @@ name: GitSecretsScan -on: [pull_request] +on: + pull_request: + branches: + - main + - '*.*' + types: [opened, reopened, synchronize] jobs: git-secret-check: diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index f5cdda74..970e2934 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -6,7 +6,7 @@ on: branches: - main - '*.*' - types: [opened, edited, reopened, synchronize] + types: [opened, reopened, synchronize] # Trigger 2: Daily scheduled run at 22:13 UTC # Schedule it a random minute because most Github Actions are scheduled @@ -108,7 +108,7 @@ jobs: with: role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} aws-region: us-east-1 - role-session-name: security-scan-${{ matrix.target }}-${{ matrix.branch }} + role-session-name: security-scan-${{ matrix.target }} - name: Publish Scan Invoked metric run: | @@ -202,7 +202,7 @@ jobs: --value 1 - name: Publish Failure Metrics - if: failure() + if: failure() && github.event_name == 'schedule' run: | echo "Job failed - publishing failure metrics" @@ -294,7 +294,7 @@ jobs: ./scripts/security-scan.sh scan-github-advisories - name: Publish Failure Metrics - if: failure() + if: failure() && github.event_name == 'schedule' run: | echo "Job failed - publishing failure metrics" From c7de5544ab4c0a7b45912f8c23c87ec3f1bc0f75 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 15:01:18 +0200 Subject: [PATCH 70/90] Change trigger to 00:13 UTC for the security scan workflow --- .github/workflows/security-scan.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 970e2934..bea3c3b9 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -8,12 +8,12 @@ on: - '*.*' types: [opened, reopened, synchronize] - # Trigger 2: Daily scheduled run at 22:13 UTC + # Trigger 2: Daily scheduled run at 00:13 UTC # Schedule it a random minute because most Github Actions are scheduled # at the start of the hour and their invocation can get delayed. # Ref: https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule schedule: - - cron: '13 22 * * *' + - cron: '13 0 * * *' # Trigger 3: Manual trigger workflow_dispatch: @@ -57,22 +57,24 @@ jobs: branches_to_scan="" workflow_name="Security Scanning" - since_date=$(date -d '24 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) + # Get previous day from 00:00 UTC to 23:59 UTC + previous_day_start=$(date -d 'yesterday' -u +%Y-%m-%dT00:00:00Z) + previous_day_end=$(date -d 'yesterday' -u +%Y-%m-%dT23:59:59Z) - echo "Checking for workflow runs since: $since_date" + echo "Checking for workflow runs from previous day: $previous_day_start to $previous_day_end" # Check each branch for recent workflow runs for branch in $branches; do echo "Checking branch: $branch" - # Get recent workflow runs for this specific workflow and branch + # Get workflow runs for this specific workflow and branch from previous day all_runs=$(gh run list --workflow="$workflow_name" --branch="$branch" --json startedAt,conclusion,headBranch --status success) - recent_runs=$(echo "$all_runs" | jq --arg since "$since_date" --arg branch "$branch" '.[] | select(.startedAt > $since and .headBranch == $branch)') + previous_day_runs=$(echo "$all_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" --arg branch "$branch" '.[] | select(.startedAt >= $start and .startedAt <= $end and .headBranch == $branch)') - if [ -n "$recent_runs" ]; then - echo "Skipping branch $branch - found recent workflow run in the last 24 hours" + if [ -n "$previous_day_runs" ]; then + echo "Skipping branch $branch - found workflow run from previous day" else - echo "Adding branch $branch to scan list - no recent workflow runs" + echo "Adding branch $branch to scan list - no workflow runs from previous day" branches_to_scan="$branches_to_scan $branch" fi done @@ -86,7 +88,7 @@ jobs: echo "Branches to scan: $json_branches" else echo "branches=[]" >> $GITHUB_OUTPUT - echo "No branches to scan - all have recent workflow runs" + echo "No branches to scan - all have workflow runs from previous day" fi fi From 7f679bfb7dc287eece9c8347418cbddcfdf91b6d Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 15:04:33 +0200 Subject: [PATCH 71/90] Add a test workflow --- .github/workflows/test-workflow.yaml | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/test-workflow.yaml diff --git a/.github/workflows/test-workflow.yaml b/.github/workflows/test-workflow.yaml new file mode 100644 index 00000000..b98084ee --- /dev/null +++ b/.github/workflows/test-workflow.yaml @@ -0,0 +1,37 @@ +name: Security Scanning + +on: + workflow_dispatch: + +jobs: + get-branches-to-scan: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + run: | + echo "Hi!" + + security-scan: + runs-on: ubuntu-latest + needs: [get-branches-to-scan] + strategy: + matrix: + target: [code-editor-sagemaker-server, code-editor-server] + steps: + - name: Assume IAM Role + run: | + sleep 5m + exit 1 + + + security-scan-global-dependencies: + runs-on: ubuntu-latest + needs: [get-branches-to-scan] + strategy: + matrix: + target: [code-editor-sagemaker-server, code-editor-server] + steps: + - name: Assume IAM Role + run: | + echo "Exiting" + exit 1 \ No newline at end of file From 1f8502b054a220e7d0d6e45107d85fa0cdb17d8d Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 15:05:36 +0200 Subject: [PATCH 72/90] Change test workflow name --- .github/workflows/test-workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-workflow.yaml b/.github/workflows/test-workflow.yaml index b98084ee..57817f76 100644 --- a/.github/workflows/test-workflow.yaml +++ b/.github/workflows/test-workflow.yaml @@ -1,4 +1,4 @@ -name: Security Scanning +name: TestWf on: workflow_dispatch: From 43b282cd12dc1823877201a67bd5136d8e90286f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 15:06:36 +0200 Subject: [PATCH 73/90] Add TestWf --- .github/workflows/test-wf.yaml | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/test-wf.yaml diff --git a/.github/workflows/test-wf.yaml b/.github/workflows/test-wf.yaml new file mode 100644 index 00000000..57817f76 --- /dev/null +++ b/.github/workflows/test-wf.yaml @@ -0,0 +1,37 @@ +name: TestWf + +on: + workflow_dispatch: + +jobs: + get-branches-to-scan: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + run: | + echo "Hi!" + + security-scan: + runs-on: ubuntu-latest + needs: [get-branches-to-scan] + strategy: + matrix: + target: [code-editor-sagemaker-server, code-editor-server] + steps: + - name: Assume IAM Role + run: | + sleep 5m + exit 1 + + + security-scan-global-dependencies: + runs-on: ubuntu-latest + needs: [get-branches-to-scan] + strategy: + matrix: + target: [code-editor-sagemaker-server, code-editor-server] + steps: + - name: Assume IAM Role + run: | + echo "Exiting" + exit 1 \ No newline at end of file From 7e346ae38c67b9464cea35b45eef59ee05abf0e9 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 15:16:55 +0200 Subject: [PATCH 74/90] Remove TestWf --- .github/workflows/test-wf.yaml | 37 ---------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 .github/workflows/test-wf.yaml diff --git a/.github/workflows/test-wf.yaml b/.github/workflows/test-wf.yaml deleted file mode 100644 index 57817f76..00000000 --- a/.github/workflows/test-wf.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: TestWf - -on: - workflow_dispatch: - -jobs: - get-branches-to-scan: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - run: | - echo "Hi!" - - security-scan: - runs-on: ubuntu-latest - needs: [get-branches-to-scan] - strategy: - matrix: - target: [code-editor-sagemaker-server, code-editor-server] - steps: - - name: Assume IAM Role - run: | - sleep 5m - exit 1 - - - security-scan-global-dependencies: - runs-on: ubuntu-latest - needs: [get-branches-to-scan] - strategy: - matrix: - target: [code-editor-sagemaker-server, code-editor-server] - steps: - - name: Assume IAM Role - run: | - echo "Exiting" - exit 1 \ No newline at end of file From d7a8c1f6d06e4b3b2726cb5ea0039ab09cac2985 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 20:44:56 +0200 Subject: [PATCH 75/90] Modify the logic for checking previously scanned branches --- .github/workflows/security-scan.yml | 331 +++++++++++++++++++++++----- 1 file changed, 275 insertions(+), 56 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index bea3c3b9..efc07f92 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -1,5 +1,8 @@ name: Security Scanning +env: + CODE_EDITOR_TARGETS: '["code-editor-sagemaker-server"]' + on: # Trigger 1: PR created on main or version branches (*.*) pull_request: @@ -22,87 +25,209 @@ jobs: get-branches-to-scan: runs-on: ubuntu-latest outputs: - branches: ${{ steps.determine-branches.outputs.branches }} + security-scan-branches: ${{ steps.determine-pr-branches.outputs.branches || steps.determine-scheduled-security-scan-branches.outputs.branches }} + global-dependencies-branches: ${{ steps.determine-pr-branches.outputs.branches || steps.determine-scheduled-global-dependencies-branches.outputs.branches }} + output-branch-name: ${{ steps.determine-pr-branches.outputs.output-branch-name || steps.determine-scheduled-branches.outputs.output-branch-name }} steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Determine branches to scan - id: determine-branches + - name: Determine branches for PR events + id: determine-pr-branches + if: github.event_name == 'pull_request' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - # For PR events, validate base branch and use head ref if valid - base_ref="${{ github.base_ref }}" - echo "Base branch: $base_ref" - - if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then - echo "Base branch matches allowed pattern (main or digit.digit)" - echo "branches=[\"${{ github.head_ref }}\"]" >> $GITHUB_OUTPUT - echo "Branches to scan: [${{ github.head_ref }}]" - else - echo "Base branch does not match allowed pattern - no branches to scan" - echo "branches=[]" >> $GITHUB_OUTPUT - fi + # For PR events, validate base branch and use head ref if valid + base_ref="${{ github.base_ref }}" + head_ref="${{ github.head_ref }}" + echo "Base branch: $base_ref" + echo "Head branch: $head_ref" + + if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then + echo "Base branch matches allowed pattern (main or digit.digit)" + echo "branches=[\"$head_ref\"]" >> $GITHUB_OUTPUT + echo "output-branch-name=$base_ref" >> $GITHUB_OUTPUT + echo "Branches to scan: [$head_ref]" + echo "Output files will use branch name: $base_ref" else - # For scheduled/manual runs, get branches and filter by recent runs - echo "Getting branches for scheduled/manual run" - - # Get main branch and all version branches (*.*) - branches=$(git branch -r | grep -E 'origin/(main|[0-9]+\.[0-9]+)' | sed 's/origin\///' | tr '\n' ' ') - echo "Found branches: $branches" - - branches_to_scan="" - workflow_name="Security Scanning" - # Get previous day from 00:00 UTC to 23:59 UTC - previous_day_start=$(date -d 'yesterday' -u +%Y-%m-%dT00:00:00Z) - previous_day_end=$(date -d 'yesterday' -u +%Y-%m-%dT23:59:59Z) - - echo "Checking for workflow runs from previous day: $previous_day_start to $previous_day_end" - - # Check each branch for recent workflow runs - for branch in $branches; do - echo "Checking branch: $branch" + echo "Base branch does not match allowed pattern - no branches to scan" + echo "branches=[]" >> $GITHUB_OUTPUT + echo "output-branch-name=" >> $GITHUB_OUTPUT + fi + + - name: Get all upstream branches + id: get-upstream-branches + if: github.event_name != 'pull_request' + run: | + # Get main branch and all version branches (*.*) + branches=$(git branch -r | grep -E 'origin/(main|[0-9]+\.[0-9]+)' | sed 's/origin\///' | tr '\n' ' ') + echo "Found upstream branches: $branches" + echo "upstream-branches=$branches" >> $GITHUB_OUTPUT + echo "output-branch-name=scheduled" >> $GITHUB_OUTPUT + + - name: Get completed workflows from previous day + id: get-completed-workflows + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + workflow_name="Security Scanning" + # Get previous day from 00:00 UTC to 23:59 UTC + previous_day_start=$(date -d 'yesterday' -u +%Y-%m-%dT00:00:00Z) + previous_day_end=$(date -d 'yesterday' -u +%Y-%m-%dT23:59:59Z) + + echo "Getting completed workflows from previous day: $previous_day_start to $previous_day_end" + + # Get all completed workflow runs from previous day + completed_runs=$(gh run list --workflow="$workflow_name" --json id,startedAt,conclusion,headBranch --status completed --limit 100) + previous_day_runs=$(echo "$completed_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" '.[] | select(.startedAt >= $start and .startedAt <= $end)') + + echo "Found completed workflow runs from previous day:" + echo "$previous_day_runs" | jq -r '.id' + + # Store workflow run IDs for artifact checking + run_ids=$(echo "$previous_day_runs" | jq -r '.id' | tr '\n' ' ') + echo "workflow-run-ids=$run_ids" >> $GITHUB_OUTPUT + + - name: Check for successful scan artifacts from previous day + id: check-scan-artifacts + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + run_ids="${{ steps.get-completed-workflows.outputs.workflow-run-ids }}" + successful_security_scan_branches="" + successful_global_dependencies_branches="" + + echo "Checking for successful scan artifacts from workflow runs: $run_ids" + + for run_id in $run_ids; do + if [ -n "$run_id" ]; then + echo "Checking artifacts for run ID: $run_id" - # Get workflow runs for this specific workflow and branch from previous day - all_runs=$(gh run list --workflow="$workflow_name" --branch="$branch" --json startedAt,conclusion,headBranch --status success) - previous_day_runs=$(echo "$all_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" --arg branch "$branch" '.[] | select(.startedAt >= $start and .startedAt <= $end and .headBranch == $branch)') + # Get artifacts for this run + artifacts=$(gh run view $run_id --json artifacts --jq '.artifacts[].name') - if [ -n "$previous_day_runs" ]; then - echo "Skipping branch $branch - found workflow run from previous day" + # Check for scan-success-branch-* artifacts + security_scan_artifacts=$(echo "$artifacts" | grep "^scan-success-branch-" || true) + global_dependencies_artifacts=$(echo "$artifacts" | grep "^global-scan-success-" || true) + + # Extract branch names from artifact names + for artifact in $security_scan_artifacts; do + branch_name=$(echo "$artifact" | sed 's/scan-success-branch-files//' | sed 's/scan-success-branch-//') + if [ -n "$branch_name" ]; then + successful_security_scan_branches="$successful_security_scan_branches $branch_name" + fi + done + + for artifact in $global_dependencies_artifacts; do + branch_name=$(echo "$artifact" | sed 's/global-scan-success-//') + if [ -n "$branch_name" ]; then + successful_global_dependencies_branches="$successful_global_dependencies_branches $branch_name" + fi + done + fi + done + + # Remove duplicates and clean up + successful_security_scan_branches=$(echo $successful_security_scan_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') + successful_global_dependencies_branches=$(echo $successful_global_dependencies_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') + + echo "Branches with successful security scans from previous day: $successful_security_scan_branches" + echo "Branches with successful global dependency scans from previous day: $successful_global_dependencies_branches" + + echo "successful-security-scan-branches=$successful_security_scan_branches" >> $GITHUB_OUTPUT + echo "successful-global-dependencies-branches=$successful_global_dependencies_branches" >> $GITHUB_OUTPUT + + - name: Determine security scan branches for scheduled runs + id: determine-scheduled-security-scan-branches + if: github.event_name != 'pull_request' + run: | + upstream_branches="${{ steps.get-upstream-branches.outputs.upstream-branches }}" + successful_branches="${{ steps.check-scan-artifacts.outputs.successful-security-scan-branches }}" + + branches_to_scan="" + + echo "Upstream branches: $upstream_branches" + echo "Successfully scanned branches from previous day: $successful_branches" + + # Check each upstream branch + for branch in $upstream_branches; do + branch=$(echo $branch | xargs) # trim whitespace + if [ -n "$branch" ]; then + # Check if this branch was successfully scanned yesterday + if echo "$successful_branches" | grep -q "\b$branch\b"; then + echo "Skipping branch $branch - found successful scan from previous day" else - echo "Adding branch $branch to scan list - no workflow runs from previous day" + echo "Adding branch $branch to security scan list - no successful scan from previous day" branches_to_scan="$branches_to_scan $branch" fi - done - - # Clean up extra spaces and convert to JSON array format - branches_to_scan=$(echo $branches_to_scan | xargs) - if [ -n "$branches_to_scan" ]; then - # Convert space-separated list to JSON array (compact format) - json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) - echo "branches=$json_branches" >> $GITHUB_OUTPUT - echo "Branches to scan: $json_branches" - else - echo "branches=[]" >> $GITHUB_OUTPUT - echo "No branches to scan - all have workflow runs from previous day" fi + done + + # Clean up and convert to JSON array + branches_to_scan=$(echo $branches_to_scan | xargs) + if [ -n "$branches_to_scan" ]; then + json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) + echo "branches=$json_branches" >> $GITHUB_OUTPUT + echo "Security scan branches to scan: $json_branches" + else + echo "branches=[]" >> $GITHUB_OUTPUT + echo "No security scan branches to scan - all have successful scans from previous day" + fi + + - name: Determine global dependencies branches for scheduled runs + id: determine-scheduled-global-dependencies-branches + if: github.event_name != 'pull_request' + run: | + upstream_branches="${{ steps.get-upstream-branches.outputs.upstream-branches }}" + successful_branches="${{ steps.check-scan-artifacts.outputs.successful-global-dependencies-branches }}" + + branches_to_scan="" + + echo "Upstream branches: $upstream_branches" + echo "Successfully scanned global dependencies branches from previous day: $successful_branches" + + # Check each upstream branch + for branch in $upstream_branches; do + branch=$(echo $branch | xargs) # trim whitespace + if [ -n "$branch" ]; then + # Check if this branch was successfully scanned yesterday + if echo "$successful_branches" | grep -q "\b$branch\b"; then + echo "Skipping branch $branch - found successful global dependencies scan from previous day" + else + echo "Adding branch $branch to global dependencies scan list - no successful scan from previous day" + branches_to_scan="$branches_to_scan $branch" + fi + fi + done + + # Clean up and convert to JSON array + branches_to_scan=$(echo $branches_to_scan | xargs) + if [ -n "$branches_to_scan" ]; then + json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) + echo "branches=$json_branches" >> $GITHUB_OUTPUT + echo "Global dependencies branches to scan: $json_branches" + else + echo "branches=[]" >> $GITHUB_OUTPUT + echo "No global dependencies branches to scan - all have successful scans from previous day" fi security-scan: runs-on: ubuntu-latest needs: [get-branches-to-scan] - if: needs.get-branches-to-scan.outputs.branches != '[]' + if: needs.get-branches-to-scan.outputs.security-scan-branches != '[]' environment: security-scanning-workflow-env permissions: id-token: write # Required for OIDC strategy: + fail-fast: false matrix: - target: [code-editor-sagemaker-server] - branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} + target: ${{ fromJson(env.CODE_EDITOR_TARGETS) }} + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.security-scan-branches) }} steps: - name: Assume IAM Role id: assume-aws-iam-role @@ -194,6 +319,23 @@ jobs: - name: Analyze SBOM Scan Results run: | ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "scan_results_paths.txt" + + - name: Create Success Indicator File + run: | + # For PR events, use base_ref as output branch name, otherwise use actual branch + if [ "${{ github.event_name }}" = "pull_request" ]; then + output_branch="${{ needs.get-branches-to-scan.outputs.output-branch-name }}" + else + output_branch="${{ matrix.branch }}" + fi + echo "PASS" > scan-success-${{ matrix.target }}-${output_branch}.txt + + - name: Upload Success Indicator File + uses: actions/upload-artifact@v4 + with: + name: scan-success-${{ matrix.target }}-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }} + path: scan-success-${{ matrix.target }}-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }}.txt + retention-days: 90 - name: Publish Scan Successful Metric run: | @@ -214,6 +356,65 @@ jobs: --metric-name "SecurityScanFailed" \ --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ --value 1 + + generate-security-scan-output: + runs-on: ubuntu-latest + needs: [security-scan] + if: always() + steps: + - name: Download all scan success files + uses: actions/download-artifact@v4 + with: + pattern: scan-success-* + merge-multiple: true + + - name: Check which branches were successful for all targets + run: | + # Parse targets and branches from environment variables and job outputs + targets_json='${{ env.CODE_EDITOR_TARGETS }}' + targets=($(echo "$targets_json" | jq -r '.[]')) + branches_json='${{ needs.security-scan.needs.get-branches-to-scan.outputs.security-scan-branches }}' + branches=($(echo "$branches_json" | jq -r '.[]')) + + echo "Targets to check: ${targets[@]}" + echo "Branches to check: ${branches[@]}" + + # Process each branch + for branch in "${branches[@]}"; do + echo "=========================================" + echo "Checking success for branch: $branch" + all_success=true + + # Check if all target success files exist for this branch + for target in "${targets[@]}"; do + success_file="scan-success-${target}-${branch}.txt" + echo "Checking for file: $success_file" + + if [ -f "$success_file" ]; then + echo "✓ Found success file for target $target on branch $branch" + else + echo "✗ Missing success file for target $target on branch $branch" + all_success=false + break + fi + done + + # Create branch success file only if all targets succeeded + if [ "$all_success" = true ]; then + echo "✓ All scans successful for branch $branch - creating branch success file" + echo "PASS" > scan-success-branch-${branch}.txt + else + echo "✗ Some scans failed for branch $branch - not creating branch success file" + fi + done + + - name: Upload Branch Success Files + uses: actions/upload-artifact@v4 + with: + name: scan-success-branch-files + path: scan-success-branch-*.txt + retention-days: 90 + if-no-files-found: ignore security-scan-global-dependencies: runs-on: ubuntu-latest @@ -222,8 +423,9 @@ jobs: permissions: id-token: write # Required for OIDC strategy: + fail-fast: false matrix: - branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.global-dependencies-branches) }} steps: - name: Assume IAM Role id: assume-aws-iam-role @@ -295,6 +497,23 @@ jobs: run: | ./scripts/security-scan.sh scan-github-advisories + - name: Create Global Success Indicator File + run: | + # For PR events, use base_ref as output branch name, otherwise use actual branch + if [ "${{ github.event_name }}" = "pull_request" ]; then + output_branch="${{ needs.get-branches-to-scan.outputs.output-branch-name }}" + else + output_branch="${{ matrix.branch }}" + fi + echo "PASS" > global-scan-success-${output_branch}.txt + + - name: Upload Global Success Indicator File + uses: actions/upload-artifact@v4 + with: + name: global-scan-success-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }} + path: global-scan-success-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }}.txt + retention-days: 90 + - name: Publish Failure Metrics if: failure() && github.event_name == 'schedule' run: | From 6e97ca256178716fa251452b12fe18cad7335371 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 20:49:04 +0200 Subject: [PATCH 76/90] Update env variables --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index efc07f92..19de878d 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -226,7 +226,7 @@ jobs: strategy: fail-fast: false matrix: - target: ${{ fromJson(env.CODE_EDITOR_TARGETS) }} + target: [code-editor-sagemaker-server] branch: ${{ fromJson(needs.get-branches-to-scan.outputs.security-scan-branches) }} steps: - name: Assume IAM Role From 0655bfb41b6d087bb6e053132fc671ff3465a41e Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 20:53:04 +0200 Subject: [PATCH 77/90] Correct context names --- .github/workflows/security-scan.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 19de878d..9151ac01 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -27,7 +27,7 @@ jobs: outputs: security-scan-branches: ${{ steps.determine-pr-branches.outputs.branches || steps.determine-scheduled-security-scan-branches.outputs.branches }} global-dependencies-branches: ${{ steps.determine-pr-branches.outputs.branches || steps.determine-scheduled-global-dependencies-branches.outputs.branches }} - output-branch-name: ${{ steps.determine-pr-branches.outputs.output-branch-name || steps.determine-scheduled-branches.outputs.output-branch-name }} + output-branch-name: ${{ steps.determine-pr-branches.outputs.output-branch-name || steps.get-upstream-branches.outputs.output-branch-name }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -359,7 +359,7 @@ jobs: generate-security-scan-output: runs-on: ubuntu-latest - needs: [security-scan] + needs: [get-branches-to-scan, security-scan] if: always() steps: - name: Download all scan success files @@ -373,7 +373,7 @@ jobs: # Parse targets and branches from environment variables and job outputs targets_json='${{ env.CODE_EDITOR_TARGETS }}' targets=($(echo "$targets_json" | jq -r '.[]')) - branches_json='${{ needs.security-scan.needs.get-branches-to-scan.outputs.security-scan-branches }}' + branches_json='${{ needs.get-branches-to-scan.outputs.security-scan-branches }}' branches=($(echo "$branches_json" | jq -r '.[]')) echo "Targets to check: ${targets[@]}" From 8874a298d52732949ed8d9759de0fcccf746ec1f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 21:30:27 +0200 Subject: [PATCH 78/90] Use databaseId, and the correct API to get artifacts --- .github/workflows/security-scan.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 9151ac01..9edceaba 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -82,14 +82,14 @@ jobs: echo "Getting completed workflows from previous day: $previous_day_start to $previous_day_end" # Get all completed workflow runs from previous day - completed_runs=$(gh run list --workflow="$workflow_name" --json id,startedAt,conclusion,headBranch --status completed --limit 100) + completed_runs=$(gh run list --workflow="$workflow_name" --json databaseId,startedAt,conclusion,headBranch --status completed --limit 100) previous_day_runs=$(echo "$completed_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" '.[] | select(.startedAt >= $start and .startedAt <= $end)') echo "Found completed workflow runs from previous day:" - echo "$previous_day_runs" | jq -r '.id' + echo "$previous_day_runs" | jq -r '.databaseId' # Store workflow run IDs for artifact checking - run_ids=$(echo "$previous_day_runs" | jq -r '.id' | tr '\n' ' ') + run_ids=$(echo "$previous_day_runs" | jq -r '.databaseId' | tr '\n' ' ') echo "workflow-run-ids=$run_ids" >> $GITHUB_OUTPUT - name: Check for successful scan artifacts from previous day @@ -109,7 +109,7 @@ jobs: echo "Checking artifacts for run ID: $run_id" # Get artifacts for this run - artifacts=$(gh run view $run_id --json artifacts --jq '.artifacts[].name') + artifacts=$(gh api /repos/${{ github.repository }}/actions/runs/$run_id/artifacts --jq '.artifacts[].name') # Check for scan-success-branch-* artifacts security_scan_artifacts=$(echo "$artifacts" | grep "^scan-success-branch-" || true) From 595f37e2b6657a4f79fe33d42f7df7e696c6ded5 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 21:39:57 +0200 Subject: [PATCH 79/90] Run security generate-security-scan-output on a matrix strategy --- .github/workflows/security-scan.yml | 64 ++++++++++++++--------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 9edceaba..88feb33e 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -361,6 +361,10 @@ jobs: runs-on: ubuntu-latest needs: [get-branches-to-scan, security-scan] if: always() + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.security-scan-branches) }} steps: - name: Download all scan success files uses: actions/download-artifact@v4 @@ -368,53 +372,47 @@ jobs: pattern: scan-success-* merge-multiple: true - - name: Check which branches were successful for all targets + - name: Check if branch was successful for all targets run: | - # Parse targets and branches from environment variables and job outputs + # Parse targets from environment variable targets_json='${{ env.CODE_EDITOR_TARGETS }}' targets=($(echo "$targets_json" | jq -r '.[]')) - branches_json='${{ needs.get-branches-to-scan.outputs.security-scan-branches }}' - branches=($(echo "$branches_json" | jq -r '.[]')) + branch="${{ matrix.branch }}" + all_success=true + echo "Checking success for branch: $branch" echo "Targets to check: ${targets[@]}" - echo "Branches to check: ${branches[@]}" - # Process each branch - for branch in "${branches[@]}"; do - echo "=========================================" - echo "Checking success for branch: $branch" - all_success=true + # Check if all target success files exist for this branch + for target in "${targets[@]}"; do + success_file="scan-success-${target}-${branch}.txt" + echo "Checking for file: $success_file" - # Check if all target success files exist for this branch - for target in "${targets[@]}"; do - success_file="scan-success-${target}-${branch}.txt" - echo "Checking for file: $success_file" - - if [ -f "$success_file" ]; then - echo "✓ Found success file for target $target on branch $branch" - else - echo "✗ Missing success file for target $target on branch $branch" - all_success=false - break - fi - done - - # Create branch success file only if all targets succeeded - if [ "$all_success" = true ]; then - echo "✓ All scans successful for branch $branch - creating branch success file" - echo "PASS" > scan-success-branch-${branch}.txt + if [ -f "$success_file" ]; then + echo "✓ Found success file for target $target on branch $branch" else - echo "✗ Some scans failed for branch $branch - not creating branch success file" + echo "✗ Missing success file for target $target on branch $branch" + all_success=false + break fi done + + # Create branch success file only if all targets succeeded + if [ "$all_success" = true ]; then + echo "✓ All scans successful for branch $branch - creating branch success file" + echo "PASS" > scan-success-branch-${branch}.txt + else + echo "✗ Some scans failed for branch $branch - not creating branch success file" + exit 1 + fi - - name: Upload Branch Success Files + - name: Upload Branch Success File + if: success() uses: actions/upload-artifact@v4 with: - name: scan-success-branch-files - path: scan-success-branch-*.txt + name: scan-success-branch-${{ matrix.branch }} + path: scan-success-branch-${{ matrix.branch }}.txt retention-days: 90 - if-no-files-found: ignore security-scan-global-dependencies: runs-on: ubuntu-latest From 53eb8dc28193e0c37e04c5096198aac4d87ec610 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 21:44:41 +0200 Subject: [PATCH 80/90] Add validation for running security scan against empty branches --- .github/workflows/security-scan.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 88feb33e..5a0b7bf6 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -219,7 +219,7 @@ jobs: security-scan: runs-on: ubuntu-latest needs: [get-branches-to-scan] - if: needs.get-branches-to-scan.outputs.security-scan-branches != '[]' + if: needs.get-branches-to-scan.outputs.security-scan-branches != '[]' && needs.get-branches-to-scan.outputs.security-scan-branches != '' environment: security-scanning-workflow-env permissions: id-token: write # Required for OIDC @@ -360,7 +360,7 @@ jobs: generate-security-scan-output: runs-on: ubuntu-latest needs: [get-branches-to-scan, security-scan] - if: always() + if: always() && needs.get-branches-to-scan.outputs.security-scan-branches != '[]' && needs.get-branches-to-scan.outputs.security-scan-branches != '' strategy: fail-fast: false matrix: @@ -417,6 +417,7 @@ jobs: security-scan-global-dependencies: runs-on: ubuntu-latest needs: [get-branches-to-scan] + if: needs.get-branches-to-scan.outputs.global-dependencies-branches != '[]' && needs.get-branches-to-scan.outputs.global-dependencies-branches != '' environment: security-scanning-workflow-env permissions: id-token: write # Required for OIDC From e38882683a66ad834747c001b2fde4e04ed99242 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 22:01:51 +0200 Subject: [PATCH 81/90] Update metrics and change from 24 hours to 2 hours ago for timing match --- .github/workflows/security-scan.yml | 86 ++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 5a0b7bf6..4cdf787a 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -1,4 +1,4 @@ -name: Security Scanning +name: Security Scan env: CODE_EDITOR_TARGETS: '["code-editor-sagemaker-server"]' @@ -75,21 +75,21 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | workflow_name="Security Scanning" - # Get previous day from 00:00 UTC to 23:59 UTC - previous_day_start=$(date -d 'yesterday' -u +%Y-%m-%dT00:00:00Z) - previous_day_end=$(date -d 'yesterday' -u +%Y-%m-%dT23:59:59Z) + # Get workflows from 2 hours ago onwards + previous_day_start=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) + previous_day_end=$(date -u +%Y-%m-%dT%H:%M:%SZ) - echo "Getting completed workflows from previous day: $previous_day_start to $previous_day_end" + echo "Getting completed workflows from 2 hours ago: $previous_day_start to $previous_day_end" - # Get all completed workflow runs from previous day + # Get all completed workflow runs from 2 hours ago onwards completed_runs=$(gh run list --workflow="$workflow_name" --json databaseId,startedAt,conclusion,headBranch --status completed --limit 100) - previous_day_runs=$(echo "$completed_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" '.[] | select(.startedAt >= $start and .startedAt <= $end)') + recent_runs=$(echo "$completed_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" '.[] | select(.startedAt >= $start and .startedAt <= $end)') - echo "Found completed workflow runs from previous day:" - echo "$previous_day_runs" | jq -r '.databaseId' + echo "Found completed workflow runs from 2 hours ago:" + echo "$recent_runs" | jq -r '.databaseId' # Store workflow run IDs for artifact checking - run_ids=$(echo "$previous_day_runs" | jq -r '.databaseId' | tr '\n' ' ') + run_ids=$(echo "$recent_runs" | jq -r '.databaseId' | tr '\n' ' ') echo "workflow-run-ids=$run_ids" >> $GITHUB_OUTPUT - name: Check for successful scan artifacts from previous day @@ -136,8 +136,8 @@ jobs: successful_security_scan_branches=$(echo $successful_security_scan_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') successful_global_dependencies_branches=$(echo $successful_global_dependencies_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') - echo "Branches with successful security scans from previous day: $successful_security_scan_branches" - echo "Branches with successful global dependency scans from previous day: $successful_global_dependencies_branches" + echo "Branches with successful security scans from 2 hours ago: $successful_security_scan_branches" + echo "Branches with successful global dependency scans from 2 hours ago: $successful_global_dependencies_branches" echo "successful-security-scan-branches=$successful_security_scan_branches" >> $GITHUB_OUTPUT echo "successful-global-dependencies-branches=$successful_global_dependencies_branches" >> $GITHUB_OUTPUT @@ -152,17 +152,17 @@ jobs: branches_to_scan="" echo "Upstream branches: $upstream_branches" - echo "Successfully scanned branches from previous day: $successful_branches" + echo "Successfully scanned branches from 2 hours ago: $successful_branches" # Check each upstream branch for branch in $upstream_branches; do branch=$(echo $branch | xargs) # trim whitespace if [ -n "$branch" ]; then - # Check if this branch was successfully scanned yesterday + # Check if this branch was successfully scanned in the last 2 hours if echo "$successful_branches" | grep -q "\b$branch\b"; then - echo "Skipping branch $branch - found successful scan from previous day" + echo "Skipping branch $branch - found successful scan from last 2 hours" else - echo "Adding branch $branch to security scan list - no successful scan from previous day" + echo "Adding branch $branch to security scan list - no successful scan from last 2 hours" branches_to_scan="$branches_to_scan $branch" fi fi @@ -176,7 +176,7 @@ jobs: echo "Security scan branches to scan: $json_branches" else echo "branches=[]" >> $GITHUB_OUTPUT - echo "No security scan branches to scan - all have successful scans from previous day" + echo "No security scan branches to scan - all have successful scans from last 2 hours" fi - name: Determine global dependencies branches for scheduled runs @@ -189,17 +189,17 @@ jobs: branches_to_scan="" echo "Upstream branches: $upstream_branches" - echo "Successfully scanned global dependencies branches from previous day: $successful_branches" + echo "Successfully scanned global dependencies branches from 2 hours ago: $successful_branches" # Check each upstream branch for branch in $upstream_branches; do branch=$(echo $branch | xargs) # trim whitespace if [ -n "$branch" ]; then - # Check if this branch was successfully scanned yesterday + # Check if this branch was successfully scanned in the last 2 hours if echo "$successful_branches" | grep -q "\b$branch\b"; then - echo "Skipping branch $branch - found successful global dependencies scan from previous day" + echo "Skipping branch $branch - found successful global dependencies scan from last 2 hours" else - echo "Adding branch $branch to global dependencies scan list - no successful scan from previous day" + echo "Adding branch $branch to global dependencies scan list - no successful scan from last 2 hours" branches_to_scan="$branches_to_scan $branch" fi fi @@ -213,7 +213,7 @@ jobs: echo "Global dependencies branches to scan: $json_branches" else echo "branches=[]" >> $GITHUB_OUTPUT - echo "No global dependencies branches to scan - all have successful scans from previous day" + echo "No global dependencies branches to scan - all have successful scans from last 2 hours" fi security-scan: @@ -242,7 +242,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ --value 1 - name: Checkout branch @@ -342,7 +342,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanSuccessful" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ --value 1 - name: Publish Failure Metrics @@ -354,7 +354,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ --value 1 generate-security-scan-output: @@ -433,6 +433,14 @@ jobs: role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} aws-region: us-east-1 role-session-name: security-scan-global-dependencies + + - name: Publish Scan Invoked metric + run: | + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "GlobalDependenciesSecurityScanInvoked" \ + --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan" \ + --value 1 - name: Checkout branch uses: actions/checkout@v4 @@ -522,5 +530,31 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan" \ + --value 1 + + handle-failures: + name: Handle Failures + runs-on: ubuntu-latest + needs: [get-branches-to-scan, generate-security-scan-output] + environment: security-scanning-workflow-env + if: failure() + permissions: + id-token: write # Required for OIDC + env: + REPOSITORY: ${{ github.repository }} + AWS_IAM_ROLE_ARN: ${{ secrets.AWS_IAM_ROLE_ARN }} + steps: + - name: Use role credentials for metrics + id: aws-creds + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_IAM_ROLE_ARN }} + aws-region: us-east-1 + - name: Report failure + run: | + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "ExecutionsFailed" \ + --dimensions "Repository=${{ env.REPOSITORY }},Workflow=SecurityScan" \ --value 1 \ No newline at end of file From 0230c120fb67896ea590e22e3bca3645a72c128f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 22:14:36 +0200 Subject: [PATCH 82/90] Update check branch to PR base ref --- .github/workflows/security-scan.yml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 4cdf787a..ddaf43dc 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -377,21 +377,28 @@ jobs: # Parse targets from environment variable targets_json='${{ env.CODE_EDITOR_TARGETS }}' targets=($(echo "$targets_json" | jq -r '.[]')) - branch="${{ matrix.branch }}" + + # For PR events, use base_ref as output branch name, otherwise use actual branch + if [ "${{ github.event_name }}" = "pull_request" ]; then + check_branch="${{ needs.get-branches-to-scan.outputs.output-branch-name }}" + else + check_branch="${{ matrix.branch }}" + fi + all_success=true - echo "Checking success for branch: $branch" + echo "Checking success for branch: $check_branch (matrix branch: ${{ matrix.branch }})" echo "Targets to check: ${targets[@]}" # Check if all target success files exist for this branch for target in "${targets[@]}"; do - success_file="scan-success-${target}-${branch}.txt" + success_file="scan-success-${target}-${check_branch}.txt" echo "Checking for file: $success_file" if [ -f "$success_file" ]; then - echo "✓ Found success file for target $target on branch $branch" + echo "✓ Found success file for target $target on branch $check_branch" else - echo "✗ Missing success file for target $target on branch $branch" + echo "✗ Missing success file for target $target on branch $check_branch" all_success=false break fi @@ -399,10 +406,10 @@ jobs: # Create branch success file only if all targets succeeded if [ "$all_success" = true ]; then - echo "✓ All scans successful for branch $branch - creating branch success file" - echo "PASS" > scan-success-branch-${branch}.txt + echo "✓ All scans successful for branch $check_branch - creating branch success file" + echo "PASS" > scan-success-branch-${check_branch}.txt else - echo "✗ Some scans failed for branch $branch - not creating branch success file" + echo "✗ Some scans failed for branch $check_branch - not creating branch success file" exit 1 fi @@ -410,8 +417,8 @@ jobs: if: success() uses: actions/upload-artifact@v4 with: - name: scan-success-branch-${{ matrix.branch }} - path: scan-success-branch-${{ matrix.branch }}.txt + name: scan-success-branch-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }} + path: scan-success-branch-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }}.txt retention-days: 90 security-scan-global-dependencies: From 9020969e20d63f053c630284dac768dc8a1c0f34 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 22:28:59 +0200 Subject: [PATCH 83/90] Update security scanning workflow --- .github/workflows/security-scan.yml | 381 +++++++++++++++++++++++----- 1 file changed, 321 insertions(+), 60 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 970e2934..117e52a7 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -1,4 +1,7 @@ -name: Security Scanning +name: Security Scan + +env: + CODE_EDITOR_TARGETS: '["code-editor-sagemaker-server"]' on: # Trigger 1: PR created on main or version branches (*.*) @@ -8,12 +11,12 @@ on: - '*.*' types: [opened, reopened, synchronize] - # Trigger 2: Daily scheduled run at 22:13 UTC + # Trigger 2: Daily scheduled run at 00:13 UTC # Schedule it a random minute because most Github Actions are scheduled # at the start of the hour and their invocation can get delayed. # Ref: https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule schedule: - - cron: '13 22 * * *' + - cron: '13 0 * * *' # Trigger 3: Manual trigger workflow_dispatch: @@ -22,85 +25,209 @@ jobs: get-branches-to-scan: runs-on: ubuntu-latest outputs: - branches: ${{ steps.determine-branches.outputs.branches }} + security-scan-branches: ${{ steps.determine-pr-branches.outputs.branches || steps.determine-scheduled-security-scan-branches.outputs.branches }} + global-dependencies-branches: ${{ steps.determine-pr-branches.outputs.branches || steps.determine-scheduled-global-dependencies-branches.outputs.branches }} + output-branch-name: ${{ steps.determine-pr-branches.outputs.output-branch-name || steps.get-upstream-branches.outputs.output-branch-name }} steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Determine branches to scan - id: determine-branches + - name: Determine branches for PR events + id: determine-pr-branches + if: github.event_name == 'pull_request' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - # For PR events, validate base branch and use head ref if valid - base_ref="${{ github.base_ref }}" - echo "Base branch: $base_ref" - - if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then - echo "Base branch matches allowed pattern (main or digit.digit)" - echo "branches=[\"${{ github.head_ref }}\"]" >> $GITHUB_OUTPUT - echo "Branches to scan: [${{ github.head_ref }}]" - else - echo "Base branch does not match allowed pattern - no branches to scan" - echo "branches=[]" >> $GITHUB_OUTPUT - fi + # For PR events, validate base branch and use head ref if valid + base_ref="${{ github.base_ref }}" + head_ref="${{ github.head_ref }}" + echo "Base branch: $base_ref" + echo "Head branch: $head_ref" + + if [[ "$base_ref" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$base_ref" == "main" ]]; then + echo "Base branch matches allowed pattern (main or digit.digit)" + echo "branches=[\"$head_ref\"]" >> $GITHUB_OUTPUT + echo "output-branch-name=$base_ref" >> $GITHUB_OUTPUT + echo "Branches to scan: [$head_ref]" + echo "Output files will use branch name: $base_ref" else - # For scheduled/manual runs, get branches and filter by recent runs - echo "Getting branches for scheduled/manual run" - - # Get main branch and all version branches (*.*) - branches=$(git branch -r | grep -E 'origin/(main|[0-9]+\.[0-9]+)' | sed 's/origin\///' | tr '\n' ' ') - echo "Found branches: $branches" - - branches_to_scan="" - workflow_name="Security Scanning" - since_date=$(date -d '24 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) - - echo "Checking for workflow runs since: $since_date" - - # Check each branch for recent workflow runs - for branch in $branches; do - echo "Checking branch: $branch" + echo "Base branch does not match allowed pattern - no branches to scan" + echo "branches=[]" >> $GITHUB_OUTPUT + echo "output-branch-name=" >> $GITHUB_OUTPUT + fi + + - name: Get all upstream branches + id: get-upstream-branches + if: github.event_name != 'pull_request' + run: | + # Get main branch and all version branches (*.*) + branches=$(git branch -r | grep -E 'origin/(main|[0-9]+\.[0-9]+)' | sed 's/origin\///' | tr '\n' ' ') + echo "Found upstream branches: $branches" + echo "upstream-branches=$branches" >> $GITHUB_OUTPUT + echo "output-branch-name=scheduled" >> $GITHUB_OUTPUT + + - name: Get completed workflows from previous day + id: get-completed-workflows + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + workflow_name="Security Scanning" + # Get workflows from 2 hours ago onwards + previous_day_start=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) + previous_day_end=$(date -u +%Y-%m-%dT%H:%M:%SZ) + + echo "Getting completed workflows from 2 hours ago: $previous_day_start to $previous_day_end" + + # Get all completed workflow runs from 2 hours ago onwards + completed_runs=$(gh run list --workflow="$workflow_name" --json databaseId,startedAt,conclusion,headBranch --status completed --limit 100) + recent_runs=$(echo "$completed_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" '.[] | select(.startedAt >= $start and .startedAt <= $end)') + + echo "Found completed workflow runs from 2 hours ago:" + echo "$recent_runs" | jq -r '.databaseId' + + # Store workflow run IDs for artifact checking + run_ids=$(echo "$recent_runs" | jq -r '.databaseId' | tr '\n' ' ') + echo "workflow-run-ids=$run_ids" >> $GITHUB_OUTPUT + + - name: Check for successful scan artifacts from previous day + id: check-scan-artifacts + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + run_ids="${{ steps.get-completed-workflows.outputs.workflow-run-ids }}" + successful_security_scan_branches="" + successful_global_dependencies_branches="" + + echo "Checking for successful scan artifacts from workflow runs: $run_ids" + + for run_id in $run_ids; do + if [ -n "$run_id" ]; then + echo "Checking artifacts for run ID: $run_id" + + # Get artifacts for this run + artifacts=$(gh api /repos/${{ github.repository }}/actions/runs/$run_id/artifacts --jq '.artifacts[].name') + + # Check for scan-success-branch-* artifacts + security_scan_artifacts=$(echo "$artifacts" | grep "^scan-success-branch-" || true) + global_dependencies_artifacts=$(echo "$artifacts" | grep "^global-scan-success-" || true) - # Get recent workflow runs for this specific workflow and branch - all_runs=$(gh run list --workflow="$workflow_name" --branch="$branch" --json startedAt,conclusion,headBranch --status success) - recent_runs=$(echo "$all_runs" | jq --arg since "$since_date" --arg branch "$branch" '.[] | select(.startedAt > $since and .headBranch == $branch)') + # Extract branch names from artifact names + for artifact in $security_scan_artifacts; do + branch_name=$(echo "$artifact" | sed 's/scan-success-branch-files//' | sed 's/scan-success-branch-//') + if [ -n "$branch_name" ]; then + successful_security_scan_branches="$successful_security_scan_branches $branch_name" + fi + done - if [ -n "$recent_runs" ]; then - echo "Skipping branch $branch - found recent workflow run in the last 24 hours" + for artifact in $global_dependencies_artifacts; do + branch_name=$(echo "$artifact" | sed 's/global-scan-success-//') + if [ -n "$branch_name" ]; then + successful_global_dependencies_branches="$successful_global_dependencies_branches $branch_name" + fi + done + fi + done + + # Remove duplicates and clean up + successful_security_scan_branches=$(echo $successful_security_scan_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') + successful_global_dependencies_branches=$(echo $successful_global_dependencies_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') + + echo "Branches with successful security scans from 2 hours ago: $successful_security_scan_branches" + echo "Branches with successful global dependency scans from 2 hours ago: $successful_global_dependencies_branches" + + echo "successful-security-scan-branches=$successful_security_scan_branches" >> $GITHUB_OUTPUT + echo "successful-global-dependencies-branches=$successful_global_dependencies_branches" >> $GITHUB_OUTPUT + + - name: Determine security scan branches for scheduled runs + id: determine-scheduled-security-scan-branches + if: github.event_name != 'pull_request' + run: | + upstream_branches="${{ steps.get-upstream-branches.outputs.upstream-branches }}" + successful_branches="${{ steps.check-scan-artifacts.outputs.successful-security-scan-branches }}" + + branches_to_scan="" + + echo "Upstream branches: $upstream_branches" + echo "Successfully scanned branches from 2 hours ago: $successful_branches" + + # Check each upstream branch + for branch in $upstream_branches; do + branch=$(echo $branch | xargs) # trim whitespace + if [ -n "$branch" ]; then + # Check if this branch was successfully scanned in the last 2 hours + if echo "$successful_branches" | grep -q "\b$branch\b"; then + echo "Skipping branch $branch - found successful scan from last 2 hours" else - echo "Adding branch $branch to scan list - no recent workflow runs" + echo "Adding branch $branch to security scan list - no successful scan from last 2 hours" branches_to_scan="$branches_to_scan $branch" fi - done - - # Clean up extra spaces and convert to JSON array format - branches_to_scan=$(echo $branches_to_scan | xargs) - if [ -n "$branches_to_scan" ]; then - # Convert space-separated list to JSON array (compact format) - json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) - echo "branches=$json_branches" >> $GITHUB_OUTPUT - echo "Branches to scan: $json_branches" - else - echo "branches=[]" >> $GITHUB_OUTPUT - echo "No branches to scan - all have recent workflow runs" fi + done + + # Clean up and convert to JSON array + branches_to_scan=$(echo $branches_to_scan | xargs) + if [ -n "$branches_to_scan" ]; then + json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) + echo "branches=$json_branches" >> $GITHUB_OUTPUT + echo "Security scan branches to scan: $json_branches" + else + echo "branches=[]" >> $GITHUB_OUTPUT + echo "No security scan branches to scan - all have successful scans from last 2 hours" + fi + + - name: Determine global dependencies branches for scheduled runs + id: determine-scheduled-global-dependencies-branches + if: github.event_name != 'pull_request' + run: | + upstream_branches="${{ steps.get-upstream-branches.outputs.upstream-branches }}" + successful_branches="${{ steps.check-scan-artifacts.outputs.successful-global-dependencies-branches }}" + + branches_to_scan="" + + echo "Upstream branches: $upstream_branches" + echo "Successfully scanned global dependencies branches from 2 hours ago: $successful_branches" + + # Check each upstream branch + for branch in $upstream_branches; do + branch=$(echo $branch | xargs) # trim whitespace + if [ -n "$branch" ]; then + # Check if this branch was successfully scanned in the last 2 hours + if echo "$successful_branches" | grep -q "\b$branch\b"; then + echo "Skipping branch $branch - found successful global dependencies scan from last 2 hours" + else + echo "Adding branch $branch to global dependencies scan list - no successful scan from last 2 hours" + branches_to_scan="$branches_to_scan $branch" + fi + fi + done + + # Clean up and convert to JSON array + branches_to_scan=$(echo $branches_to_scan | xargs) + if [ -n "$branches_to_scan" ]; then + json_branches=$(echo "$branches_to_scan" | tr ' ' '\n' | jq -R . | jq -s -c .) + echo "branches=$json_branches" >> $GITHUB_OUTPUT + echo "Global dependencies branches to scan: $json_branches" + else + echo "branches=[]" >> $GITHUB_OUTPUT + echo "No global dependencies branches to scan - all have successful scans from last 2 hours" fi security-scan: runs-on: ubuntu-latest needs: [get-branches-to-scan] - if: needs.get-branches-to-scan.outputs.branches != '[]' + if: needs.get-branches-to-scan.outputs.security-scan-branches != '[]' && needs.get-branches-to-scan.outputs.security-scan-branches != '' environment: security-scanning-workflow-env permissions: id-token: write # Required for OIDC strategy: + fail-fast: false matrix: target: [code-editor-sagemaker-server] - branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.security-scan-branches) }} steps: - name: Assume IAM Role id: assume-aws-iam-role @@ -115,7 +242,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ --value 1 - name: Checkout branch @@ -192,13 +319,30 @@ jobs: - name: Analyze SBOM Scan Results run: | ./scripts/security-scan.sh analyze-results "${{ matrix.target }}" "scan_results_paths.txt" + + - name: Create Success Indicator File + run: | + # For PR events, use base_ref as output branch name, otherwise use actual branch + if [ "${{ github.event_name }}" = "pull_request" ]; then + output_branch="${{ needs.get-branches-to-scan.outputs.output-branch-name }}" + else + output_branch="${{ matrix.branch }}" + fi + echo "PASS" > scan-success-${{ matrix.target }}-${output_branch}.txt + + - name: Upload Success Indicator File + uses: actions/upload-artifact@v4 + with: + name: scan-success-${{ matrix.target }}-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }} + path: scan-success-${{ matrix.target }}-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }}.txt + retention-days: 90 - name: Publish Scan Successful Metric run: | aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanSuccessful" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ --value 1 - name: Publish Failure Metrics @@ -210,18 +354,84 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ --value 1 + + generate-security-scan-output: + runs-on: ubuntu-latest + needs: [get-branches-to-scan, security-scan] + if: always() && needs.get-branches-to-scan.outputs.security-scan-branches != '[]' && needs.get-branches-to-scan.outputs.security-scan-branches != '' + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.security-scan-branches) }} + steps: + - name: Download all scan success files + uses: actions/download-artifact@v4 + with: + pattern: scan-success-* + merge-multiple: true + + - name: Check if branch was successful for all targets + run: | + # Parse targets from environment variable + targets_json='${{ env.CODE_EDITOR_TARGETS }}' + targets=($(echo "$targets_json" | jq -r '.[]')) + + # For PR events, use base_ref as output branch name, otherwise use actual branch + if [ "${{ github.event_name }}" = "pull_request" ]; then + check_branch="${{ needs.get-branches-to-scan.outputs.output-branch-name }}" + else + check_branch="${{ matrix.branch }}" + fi + + all_success=true + + echo "Checking success for branch: $check_branch (matrix branch: ${{ matrix.branch }})" + echo "Targets to check: ${targets[@]}" + + # Check if all target success files exist for this branch + for target in "${targets[@]}"; do + success_file="scan-success-${target}-${check_branch}.txt" + echo "Checking for file: $success_file" + + if [ -f "$success_file" ]; then + echo "✓ Found success file for target $target on branch $check_branch" + else + echo "✗ Missing success file for target $target on branch $check_branch" + all_success=false + break + fi + done + + # Create branch success file only if all targets succeeded + if [ "$all_success" = true ]; then + echo "✓ All scans successful for branch $check_branch - creating branch success file" + echo "PASS" > scan-success-branch-${check_branch}.txt + else + echo "✗ Some scans failed for branch $check_branch - not creating branch success file" + exit 1 + fi + + - name: Upload Branch Success File + if: success() + uses: actions/upload-artifact@v4 + with: + name: scan-success-branch-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }} + path: scan-success-branch-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }}.txt + retention-days: 90 security-scan-global-dependencies: runs-on: ubuntu-latest needs: [get-branches-to-scan] + if: needs.get-branches-to-scan.outputs.global-dependencies-branches != '[]' && needs.get-branches-to-scan.outputs.global-dependencies-branches != '' environment: security-scanning-workflow-env permissions: id-token: write # Required for OIDC strategy: + fail-fast: false matrix: - branch: ${{ fromJson(needs.get-branches-to-scan.outputs.branches) }} + branch: ${{ fromJson(needs.get-branches-to-scan.outputs.global-dependencies-branches) }} steps: - name: Assume IAM Role id: assume-aws-iam-role @@ -230,6 +440,14 @@ jobs: role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} aws-region: us-east-1 role-session-name: security-scan-global-dependencies + + - name: Publish Scan Invoked metric + run: | + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "GlobalDependenciesSecurityScanInvoked" \ + --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan" \ + --value 1 - name: Checkout branch uses: actions/checkout@v4 @@ -293,6 +511,23 @@ jobs: run: | ./scripts/security-scan.sh scan-github-advisories + - name: Create Global Success Indicator File + run: | + # For PR events, use base_ref as output branch name, otherwise use actual branch + if [ "${{ github.event_name }}" = "pull_request" ]; then + output_branch="${{ needs.get-branches-to-scan.outputs.output-branch-name }}" + else + output_branch="${{ matrix.branch }}" + fi + echo "PASS" > global-scan-success-${output_branch}.txt + + - name: Upload Global Success Indicator File + uses: actions/upload-artifact@v4 + with: + name: global-scan-success-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }} + path: global-scan-success-${{ github.event_name == 'pull_request' && needs.get-branches-to-scan.outputs.output-branch-name || matrix.branch }}.txt + retention-days: 90 + - name: Publish Failure Metrics if: failure() && github.event_name == 'schedule' run: | @@ -302,5 +537,31 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScanning" \ + --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan" \ + --value 1 + + handle-failures: + name: Handle Failures V1 + runs-on: ubuntu-latest + needs: [get-branches-to-scan, generate-security-scan-output] + environment: security-scanning-workflow-env + if: failure() + permissions: + id-token: write # Required for OIDC + env: + REPOSITORY: ${{ github.repository }} + AWS_IAM_ROLE_ARN: ${{ secrets.AWS_IAM_ROLE_ARN }} + steps: + - name: Use role credentials for metrics + id: aws-creds + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_IAM_ROLE_ARN }} + aws-region: us-east-1 + - name: Report failure + run: | + aws cloudwatch put-metric-data \ + --namespace "GitHub/Workflows" \ + --metric-name "ExecutionsFailed" \ + --dimensions "Repository=${{ env.REPOSITORY }},Workflow=SecurityScan" \ --value 1 \ No newline at end of file From db8b82ad1de1613da1ec3d873eef8773f26cde52 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 22:31:30 +0200 Subject: [PATCH 84/90] Rename workflow --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 117e52a7..cf95f552 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -74,7 +74,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - workflow_name="Security Scanning" + workflow_name="Security Scan" # Get workflows from 2 hours ago onwards previous_day_start=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) previous_day_end=$(date -u +%Y-%m-%dT%H:%M:%SZ) From f4c75c297e65d79abe8a0a419a3b96814a3245bd Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 22:31:56 +0200 Subject: [PATCH 85/90] Change workflow name --- .github/workflows/security-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ddaf43dc..d2847ede 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -74,7 +74,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - workflow_name="Security Scanning" + workflow_name="Security Scan" # Get workflows from 2 hours ago onwards previous_day_start=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) previous_day_end=$(date -u +%Y-%m-%dT%H:%M:%SZ) From e8ae7f2707e7b29243fc0b2f910c4d4b22225b70 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 22:49:42 +0200 Subject: [PATCH 86/90] Address PR comments --- .github/workflows/security-scan.yml | 14 +++++++------- scripts/security-scan.sh | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index d2847ede..87e61b7f 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -235,14 +235,14 @@ jobs: with: role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} aws-region: us-east-1 - role-session-name: security-scan-${{ matrix.target }} + role-session-name: security-scan-${{ matrix.target }}-${{matrix.branch}} - name: Publish Scan Invoked metric run: | aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanInvoked" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }},Branch=${{matrix.branch}}" \ --value 1 - name: Checkout branch @@ -342,7 +342,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanSuccessful" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }},Branch=${{matrix.branch}}" \ --value 1 - name: Publish Failure Metrics @@ -354,7 +354,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }}" \ + --dimensions "Repository=${{ github.repository }},Workflow=SecurityScan,Target=${{ matrix.target }},Branch=${{matrix.branch}}" \ --value 1 generate-security-scan-output: @@ -439,14 +439,14 @@ jobs: with: role-to-assume: ${{ secrets.AWS_IAM_ROLE_ARN }} aws-region: us-east-1 - role-session-name: security-scan-global-dependencies + role-session-name: security-scan-global-dependencies-${{matrix.branch}} - name: Publish Scan Invoked metric run: | aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "GlobalDependenciesSecurityScanInvoked" \ - --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan" \ + --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan,Branch=${{matrix.branch}}" \ --value 1 - name: Checkout branch @@ -537,7 +537,7 @@ jobs: aws cloudwatch put-metric-data \ --namespace "GitHub/Workflows" \ --metric-name "SecurityScanFailed" \ - --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan" \ + --dimensions "Repository=${{ github.repository }},Workflow=GlobalDependenciesSecurityScan,Branch=${{matrix.branch}}" \ --value 1 handle-failures: diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index 253dc19b..7b11763a 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -38,13 +38,13 @@ scan_main_dependencies() { # Check if directory exists and has package-lock.json if [ ! -d "$check_dir" ]; then - echo "Warning: Directory $check_dir does not exist, skipping..." - continue + echo "Error: Directory $check_dir does not exist." + exit 1 fi if [ ! -f "$check_dir/package-lock.json" ]; then - echo "Warning: No package-lock.json found in $check_dir, skipping..." - continue + echo "Error: No package-lock.json found in $check_dir." + exit 1 fi # Generate SBOM for this directory From ba9743aecc47efccce2f887007daaea869740d9f Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 22:55:38 +0200 Subject: [PATCH 87/90] Use .packageversionrc file to refer to versions --- .github/workflows/security-scan.yml | 5 ++--- .packageversionrc | 2 ++ .../oss-attribution-generator-version.txt | 1 - config/pinned-versions/semver-version.txt | 1 - scripts/generate-oss-attribution.sh | 12 ++++++------ 5 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 .packageversionrc delete mode 100644 config/pinned-versions/oss-attribution-generator-version.txt delete mode 100644 config/pinned-versions/semver-version.txt diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 87e61b7f..d988888f 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -469,11 +469,10 @@ jobs: npm i -g @cyclonedx/cyclonedx-npm echo "Installing OSS Attribution Generator" - oss_attribution_version=$(cat config/pinned-versions/oss-attribution-generator-version.txt) - npm i -g @electrovir/oss-attribution-generator@$oss_attribution_version + source .packageversionrc + npm i -g @electrovir/oss-attribution-generator@$oss_attribution_generator_version echo "Installing semver" - semver_version=$(cat config/pinned-versions/semver-version.txt) npm i -g semver@$semver_version echo "Installing Syft for SBOM generation" diff --git a/.packageversionrc b/.packageversionrc new file mode 100644 index 00000000..8fbe179d --- /dev/null +++ b/.packageversionrc @@ -0,0 +1,2 @@ +oss_attribution_generator_version=2.0.0 +semver_version=7.7.2 \ No newline at end of file diff --git a/config/pinned-versions/oss-attribution-generator-version.txt b/config/pinned-versions/oss-attribution-generator-version.txt deleted file mode 100644 index 359a5b95..00000000 --- a/config/pinned-versions/oss-attribution-generator-version.txt +++ /dev/null @@ -1 +0,0 @@ -2.0.0 \ No newline at end of file diff --git a/config/pinned-versions/semver-version.txt b/config/pinned-versions/semver-version.txt deleted file mode 100644 index 94204632..00000000 --- a/config/pinned-versions/semver-version.txt +++ /dev/null @@ -1 +0,0 @@ -7.7.2 \ No newline at end of file diff --git a/scripts/generate-oss-attribution.sh b/scripts/generate-oss-attribution.sh index 52c527be..9a1c0da3 100755 --- a/scripts/generate-oss-attribution.sh +++ b/scripts/generate-oss-attribution.sh @@ -92,10 +92,10 @@ generate_oss_attribution() { check_unapproved_licenses "$target" "$BUILD_SRC_DIR" fi - # Read OSS attribution generator version from file - local oss_attribution_version=$(cat "$ROOT_DIR/config/pinned-versions/oss-attribution-generator-version.txt") + # Read OSS attribution generator version from packageversionrc + source "$ROOT_DIR/.packageversionrc" - npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_version -- generate-attribution --baseDir "$BUILD_SRC_DIR" --outputDir "$oss_attribution_dir" + npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_generator_version -- generate-attribution --baseDir "$BUILD_SRC_DIR" --outputDir "$oss_attribution_dir" attribution_licenses=$(cat "$oss_attribution_dir/attribution.txt") read_status=0 @@ -170,10 +170,10 @@ generate_unified_oss_attribution() { echo "Generating unified OSS attribution for all targets" mkdir -p "$BUILD_DIR/private/oss-attribution" - # Read OSS attribution generator version from file - local oss_attribution_version=$(cat "$ROOT_DIR/config/pinned-versions/oss-attribution-generator-version.txt") + # Read OSS attribution generator version from packageversionrc + source "$ROOT_DIR/.packageversionrc" - npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_version -- generate-attribution \ + npx --yes --package @electrovir/oss-attribution-generator@$oss_attribution_generator_version -- generate-attribution \ -b "${target_dirs[0]}" "${target_dirs[1]}" "${target_dirs[2]}" "${target_dirs[3]}" \ --outputDir "$BUILD_DIR/private/oss-attribution" From 9a6c3f028ff7b3b72f72fdfd279ac65c18fd2a73 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 23:03:22 +0200 Subject: [PATCH 88/90] Intentional failure --- .github/workflows/security-scan.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index d988888f..ffe2218c 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -253,6 +253,11 @@ jobs: - name: Update security scan script from main run: | + # Check if this is branch 3.1 + if [ "${{ matrix.branch }}" = "3.1" ]; then + echo "Hi!" + fi + # Older branches may not have the latest versions of the # security scan scripts. So we download the latest one from main echo "Downloading latest security-scan.sh script from main branch" @@ -457,6 +462,11 @@ jobs: - name: Update security scan script from main run: | + # Check if this is branch 3.1 + if [ "${{ matrix.branch }}" = "3.1" ]; then + echo "Hi!" + fi + # Older branches may not have the latest versions of the # security scan scripts. So we download the latest one from main echo "Downloading latest security-scan.sh script from main branch" From 70ed16f4dd7a868e6c9d7401d83bd56e46d8b58b Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 23:09:43 +0200 Subject: [PATCH 89/90] Intentional failure exit --- .github/workflows/security-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ffe2218c..11a67c96 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -255,7 +255,7 @@ jobs: run: | # Check if this is branch 3.1 if [ "${{ matrix.branch }}" = "3.1" ]; then - echo "Hi!" + exit 1 fi # Older branches may not have the latest versions of the @@ -464,7 +464,7 @@ jobs: run: | # Check if this is branch 3.1 if [ "${{ matrix.branch }}" = "3.1" ]; then - echo "Hi!" + exit 1 fi # Older branches may not have the latest versions of the From a51a23fc2fe32f7300c2d8a38c268a1cb32a7ca7 Mon Sep 17 00:00:00 2001 From: Sachin Hulyalkar Date: Mon, 1 Sep 2025 23:24:13 +0200 Subject: [PATCH 90/90] Remove intention failure and restore previous day range --- .github/workflows/security-scan.yml | 50 ++++++++++++----------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 11a67c96..e4918170 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -75,17 +75,17 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | workflow_name="Security Scan" - # Get workflows from 2 hours ago onwards - previous_day_start=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%SZ) - previous_day_end=$(date -u +%Y-%m-%dT%H:%M:%SZ) + # Get workflows from previous day (00:00 UTC to 23:59 UTC) + previous_day_start=$(date -d 'yesterday' -u +%Y-%m-%dT00:00:00Z) + previous_day_end=$(date -d 'yesterday' -u +%Y-%m-%dT23:59:59Z) - echo "Getting completed workflows from 2 hours ago: $previous_day_start to $previous_day_end" + echo "Getting completed workflows from previous day: $previous_day_start to $previous_day_end" - # Get all completed workflow runs from 2 hours ago onwards + # Get all completed workflow runs from previous day completed_runs=$(gh run list --workflow="$workflow_name" --json databaseId,startedAt,conclusion,headBranch --status completed --limit 100) recent_runs=$(echo "$completed_runs" | jq --arg start "$previous_day_start" --arg end "$previous_day_end" '.[] | select(.startedAt >= $start and .startedAt <= $end)') - echo "Found completed workflow runs from 2 hours ago:" + echo "Found completed workflow runs from previous day:" echo "$recent_runs" | jq -r '.databaseId' # Store workflow run IDs for artifact checking @@ -136,8 +136,8 @@ jobs: successful_security_scan_branches=$(echo $successful_security_scan_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') successful_global_dependencies_branches=$(echo $successful_global_dependencies_branches | tr ' ' '\n' | sort -u | tr '\n' ' ') - echo "Branches with successful security scans from 2 hours ago: $successful_security_scan_branches" - echo "Branches with successful global dependency scans from 2 hours ago: $successful_global_dependencies_branches" + echo "Branches with successful security scans from previous day: $successful_security_scan_branches" + echo "Branches with successful global dependency scans from previous day: $successful_global_dependencies_branches" echo "successful-security-scan-branches=$successful_security_scan_branches" >> $GITHUB_OUTPUT echo "successful-global-dependencies-branches=$successful_global_dependencies_branches" >> $GITHUB_OUTPUT @@ -152,17 +152,17 @@ jobs: branches_to_scan="" echo "Upstream branches: $upstream_branches" - echo "Successfully scanned branches from 2 hours ago: $successful_branches" + echo "Successfully scanned branches from previous day: $successful_branches" # Check each upstream branch for branch in $upstream_branches; do branch=$(echo $branch | xargs) # trim whitespace if [ -n "$branch" ]; then - # Check if this branch was successfully scanned in the last 2 hours + # Check if this branch was successfully scanned in the previous day if echo "$successful_branches" | grep -q "\b$branch\b"; then - echo "Skipping branch $branch - found successful scan from last 2 hours" + echo "Skipping branch $branch - found successful scan from previous day" else - echo "Adding branch $branch to security scan list - no successful scan from last 2 hours" + echo "Adding branch $branch to security scan list - no successful scan from previous day" branches_to_scan="$branches_to_scan $branch" fi fi @@ -176,7 +176,7 @@ jobs: echo "Security scan branches to scan: $json_branches" else echo "branches=[]" >> $GITHUB_OUTPUT - echo "No security scan branches to scan - all have successful scans from last 2 hours" + echo "No security scan branches to scan - all have successful scans from previous day" fi - name: Determine global dependencies branches for scheduled runs @@ -189,17 +189,17 @@ jobs: branches_to_scan="" echo "Upstream branches: $upstream_branches" - echo "Successfully scanned global dependencies branches from 2 hours ago: $successful_branches" + echo "Successfully scanned global dependencies branches from previous day: $successful_branches" # Check each upstream branch for branch in $upstream_branches; do branch=$(echo $branch | xargs) # trim whitespace if [ -n "$branch" ]; then - # Check if this branch was successfully scanned in the last 2 hours + # Check if this branch was successfully scanned in the previous day if echo "$successful_branches" | grep -q "\b$branch\b"; then - echo "Skipping branch $branch - found successful global dependencies scan from last 2 hours" + echo "Skipping branch $branch - found successful global dependencies scan from previous day" else - echo "Adding branch $branch to global dependencies scan list - no successful scan from last 2 hours" + echo "Adding branch $branch to global dependencies scan list - no successful scan from previous day" branches_to_scan="$branches_to_scan $branch" fi fi @@ -213,7 +213,7 @@ jobs: echo "Global dependencies branches to scan: $json_branches" else echo "branches=[]" >> $GITHUB_OUTPUT - echo "No global dependencies branches to scan - all have successful scans from last 2 hours" + echo "No global dependencies branches to scan - all have successful scans from previous day" fi security-scan: @@ -253,11 +253,6 @@ jobs: - name: Update security scan script from main run: | - # Check if this is branch 3.1 - if [ "${{ matrix.branch }}" = "3.1" ]; then - exit 1 - fi - # Older branches may not have the latest versions of the # security scan scripts. So we download the latest one from main echo "Downloading latest security-scan.sh script from main branch" @@ -461,12 +456,7 @@ jobs: submodules: recursive - name: Update security scan script from main - run: | - # Check if this is branch 3.1 - if [ "${{ matrix.branch }}" = "3.1" ]; then - exit 1 - fi - + run: | # Older branches may not have the latest versions of the # security scan scripts. So we download the latest one from main echo "Downloading latest security-scan.sh script from main branch" @@ -554,7 +544,7 @@ jobs: runs-on: ubuntu-latest needs: [get-branches-to-scan, generate-security-scan-output] environment: security-scanning-workflow-env - if: failure() + if: failure() && github.event_name == 'schedule' permissions: id-token: write # Required for OIDC env: