datadog updates

.github/workflows/build-and-test.yml

@@ -0,0 +1,159 @@
+name: build-and-test
+
+# Build the datadog provider from the pinned v1 + v2 specs and run every
+# credential-free test layer on each push / PR; the live smoke suite runs
+# only where the Datadog secrets are configured; a scheduled spec-drift job
+# diffs the upstream specs against the pin and opens an issue when they move.
+
+on:
+  push:
+    branches: [main, 'feature/**']
+    paths-ignore: ['website/**']
+  pull_request:
+    branches: [main]
+    paths-ignore: ['website/**']
+  schedule:
+    - cron: '23 3 * * 1'   # weekly spec-drift check (Monday 03:23 UTC)
+  workflow_dispatch:
+
+jobs:
+  build-and-test:
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      # puts the latest stackql on PATH; bin/start-server.sh and the test
+      # runners resolve it from there
+      - name: Install stackql
+        uses: stackql/setup-stackql@v2
+
+      - name: stackql version
+        run: stackql --version
+
+      # Drift is reported, not fatal, here: the committed pin is what gets
+      # built and tested. The spec-drift job below opens the review issue.
+      - name: Verify the pinned specs against upstream (warns on drift)
+        run: |
+          if ! make fetch-spec; then
+            echo "::warning title=Datadog spec drift::The upstream v1/v2 specs no longer match provider-dev/config/spec_pin.json - run 'make refresh-spec && make build && make test' and review the diff. Building from the committed pin."
+            git checkout -- provider-dev/downloaded provider-dev/config/spec_pin.json
+          fi
+
+      - name: Build provider from the pinned specs
+        run: make split normalize mappings generate
+
+      - name: Fail on uncommitted generation drift
+        run: |
+          git add -N .
+          if ! git diff --quiet -- provider-dev/openapi provider-dev/config; then
+            echo "Generated output differs from the committed artifacts - run 'make build' and commit."
+            git diff --stat -- provider-dev/openapi provider-dev/config
+            exit 1
+          fi
+
+      - name: Offline validation
+        run: make test-offline
+
+      - name: Meta-route tests
+        run: make test-meta
+
+      - name: Generate docs (sanity - the site build runs on the web workflows)
+        run: make docs
+
+  smoke:
+    # secret-gated live smoke suite; skipped with a notice when the secrets
+    # are not configured (forks, PRs from outside). Everything it creates is
+    # named stackql-smoke-* and deleted within the run.
+    runs-on: ubuntu-latest
+    needs: build-and-test
+    if: github.event_name != 'pull_request' && github.event_name != 'schedule'
+    env:
+      DD_API_KEY: ${{ secrets.DD_API_KEY }}
+      DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
+      DD_SITE: ${{ secrets.DD_SITE }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 20
+          cache: npm
+
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Live smoke suite (local provider)
+        if: env.DD_API_KEY != ''
+        run: make smoke
+
+      - name: Live smoke skipped (no credentials)
+        if: env.DD_API_KEY == ''
+        run: |
+          echo "::notice title=Live smoke suite skipped::DD_API_KEY / DD_APP_KEY secrets are not configured - the live smoke suite did not run. Credential-free coverage still ran via the offline and meta-route suites."
+
+  spec-drift:
+    # Diff the upstream specs against the pin on a schedule and on demand,
+    # and open an issue when they move.
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Fetch and compare against the pin
+        id: drift
+        run: |
+          set +e
+          make fetch-spec > fetch.log 2>&1
+          rc=$?
+          set -e
+          cat fetch.log
+          if [ "$rc" -eq 0 ]; then
+            echo "drift=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "drift=true" >> "$GITHUB_OUTPUT"
+            npm run fetch-spec -- --update > /dev/null 2>&1 || true
+            git diff --stat -- provider-dev/downloaded provider-dev/config/spec_pin.json > drift.txt || true
+            cat drift.txt
+          fi
+
+      - name: Report drift
+        if: steps.drift.outputs.drift == 'true'
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const fs = require('fs');
+            const stat = fs.existsSync('drift.txt') ? fs.readFileSync('drift.txt', 'utf8') : '(diff unavailable)';
+            const title = 'Datadog spec drift detected';
+            const body = 'The upstream Datadog v1/v2 OpenAPI specs no longer match the pin in provider-dev/config/spec_pin.json.\n\n```\n' + stat + '\n```\nRun `make refresh-spec && make build && make test`, review the generated diff (new operations in provider-dev/config/operation_inventory.csv, changed schemas), and commit.';
+            await core.summary.addHeading(title).addCodeBlock(stat).write();
+            const { data: issues } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo, state: 'open', labels: 'spec-drift'
+            });
+            if (issues.some(i => i.title === title)) return;
+            await github.rest.issues.create({
+              owner: context.repo.owner, repo: context.repo.repo, title, body, labels: ['spec-drift']
+            });

.github/workflows/prod-web-deploy.yml

@@ -1,58 +1,58 @@
-name: Deploy to GitHub Pages
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - 'website/**'      
-
-jobs:
-  build:
-    name: Build Docusaurus
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: yarn
-          cache-dependency-path: website/yarn.lock
-
-      - name: Install dependencies
-        run: yarn install --frozen-lockfile
-        working-directory: website
-
-      - name: Build website
-        run: yarn build
-        working-directory: website
-
-      - name: Upload Build Artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: website/build # Ensure the path is correctly set to the Docusaurus build output
-
-  deploy:
-    name: Deploy to GitHub Pages
-    needs: build
-
-    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
-    permissions:
-      pages: write # to deploy to Pages
-      id-token: write # to verify the deployment originates from an appropriate source
-
-    # Deploy to the github-pages environment
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-
-    runs-on: ubuntu-latest
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
-        with:
-          working-directory: website/build # Ensures the correct directory is used for deployment
+name: Deploy to GitHub Pages
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'website/**'      
+
+jobs:
+  build:
+    name: Build Docusaurus
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 20
+          cache: yarn
+          cache-dependency-path: website/yarn.lock
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+        working-directory: website
+
+      - name: Build website
+        run: yarn build
+        working-directory: website
+
+      - name: Upload Build Artifact
+        uses: actions/upload-pages-artifact@v5
+        with:
+          path: website/build # Ensure the path is correctly set to the Docusaurus build output
+
+  deploy:
+    name: Deploy to GitHub Pages
+    needs: build
+
+    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
+    permissions:
+      pages: write # to deploy to Pages
+      id-token: write # to verify the deployment originates from an appropriate source
+
+    # Deploy to the github-pages environment
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v5
+        with:
+          working-directory: website/build # Ensures the correct directory is used for deployment

.github/workflows/test-web-deploy.yml

@@ -1,31 +1,31 @@
-name: Test deployment
-
-on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - 'website/**'
-      
-jobs:
-  test-deploy:
-    name: Test deployment
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: yarn
-          cache-dependency-path: website/yarn.lock
-
-      - name: Install dependencies
-        run: yarn install --frozen-lockfile
-        working-directory: website
-
-      - name: Test build website
-        run: yarn build
+name: Test deployment
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'website/**'
+
+jobs:
+  test-deploy:
+    name: Test deployment
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 20
+          cache: yarn
+          cache-dependency-path: website/yarn.lock
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+        working-directory: website
+
+      - name: Test build website
+        run: yarn build
         working-directory: website

.gitignore

@@ -84,3 +84,11 @@ nohup.out
 **/test-output/
 # Ignore everything in source directory except .gitkeep
 source/*
+
+# derived build artifacts (merged spec, split source specs are regenerated by make)
+provider-dev/build/
+provider-dev/source/*.yaml
+provider-dev/source/*.json
+website/.shared-config/
+tests/.venv/
+tests/__pycache__/