diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7d24e084d40..9417357aac9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,8 +15,9 @@ jobs: # # The two paths build different schemas (migrations add triggers, checks and NOT VALID # constraints that `db:push` does not), so each runs the whole suite. Vitest splits each - # suite's files across four shards; a shard runs its files one at a time against its own - # database. Files run serially, so a shard barely uses more than one core: 4 vCPU is enough. + # suite's files across four shards by measured duration (DurationBalancedSequencer in vitest.shared.ts); a + # shard runs its files one at a time against its own database. Files run serially, so a shard + # barely uses more than one core: 4 vCPU is enough. name: integration (${{ matrix.provision }}, ${{ matrix.shard }}/4) runs-on: &runner-4vcpu ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 20 @@ -272,6 +273,23 @@ jobs: working-directory: packages/db run: bun run db:migrate + # Turbopack's dev cache turns the spec's route warm-up from a cold compile (~4 min) into a + # restore. It is content-addressed, so a pull request's changed modules still recompile; the + # key carries the installed Next version so an upgrade starts from an empty cache, and the + # event and fork segments keep untrusted runs off the cache trusted runs read. + - name: Resolve Turbopack dev cache key + id: next-cache + run: echo "key=${GITHUB_REPOSITORY}-next-dev-desktop-live-${GITHUB_EVENT_NAME}${FORK_SUFFIX}-$(jq -r .version node_modules/next/package.json)" >> "$GITHUB_OUTPUT" + env: + FORK_SUFFIX: ${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + + - name: Mount Turbopack dev cache + uses: ./.github/actions/cache + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ steps.next-cache.outputs.key }} + path: ./apps/sim/.next/dev + # Chat switches, Stop, sign-out, approval and the flag-off foreground round trip. The spec # runs the recording proxy (the app's public origin) and the stand-in worker. - name: Verify desktop tools in the Electron app against a local app @@ -292,39 +310,84 @@ jobs: READY_TIMEOUT_SECONDS: 300 run: | report_dir="$RUNNER_TEMP/e2e" + next_log="$report_dir/desktop-tools-next.log" mkdir -p "$report_dir" - sudo apt-get update -q - sudo apt-get install -yq xvfb libgtk-3-0t64 libnss3 libasound2t64 libgbm1 libxss1 \ - libxtst6 libatk-bridge2.0-0t64 libxkbcommon0 > /dev/null - # Bundle only: `bun run build` also fetches the macOS node-pty prebuilds for packaging, - # which a Linux run does not use. - (cd apps/desktop && bun run scripts/build.ts) # Each app runs in its own session under an E2E_APP tag, and stop-session.sh returns once # every process it started has exited. realtime_tag="desktop-realtime-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$" server_tag="desktop-tools-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$" + # Keeps the restored cache under the cap the dev scripts apply locally. + (cd apps/sim && bun run dev:cache:cap) + start_sim() { + (cd apps/sim && E2E_APP="$server_tag" exec setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \ + --port 3021 >> "$next_log" 2>&1) & + server_pid=$! + } + # A corrupted cache aborts Turbopack instead of falling back. Only then is the shared cache + # dropped: a failing test must not cost every later run its warm cache. + cache_broken() { + grep -qiE 'cache corruption|turbopack.*panic|panicked' "$next_log" 2>/dev/null + } + # The cache directory is a mount point: empty it rather than remove it. Absolute, because + # the EXIT trap runs after the step has moved into apps/desktop. + clear_cache() { + find "$GITHUB_WORKSPACE/apps/sim/.next/dev" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + } + # SIGINT first: `next dev` SIGKILLs its server 100ms after SIGTERM, which discards a cache + # write in flight. stop-session.sh then removes anything still running. + # SIGINT is best-effort; the cleanup always runs, since workers and the detached telemetry + # flush can outlive a server that has already exited. + stop_sim() { + if kill -INT "$server_pid" 2>/dev/null; then + for _ in $(seq 1 30); do kill -0 "$server_pid" 2>/dev/null || break; sleep 1; done + fi + bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$server_tag" + } (cd apps/realtime && PORT=3023 SIM_DB_ROLE=realtime ALLOWED_ORIGINS="$NEXT_PUBLIC_APP_URL" \ E2E_APP="$realtime_tag" exec setsid bun src/index.ts > "$report_dir/desktop-tools-realtime.log" 2>&1) & realtime_pid=$! - (cd apps/sim && E2E_APP="$server_tag" exec setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \ - --port 3021 > "$report_dir/desktop-tools-next.log" 2>&1) & - server_pid=$! + start_sim finish() { status=$? - bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$server_tag" || status=1 + stop_sim || status=1 bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$realtime_pid" "$realtime_tag" || status=1 wait "$server_pid" "$realtime_pid" 2>/dev/null || true + if cache_broken; then + echo "::warning::Turbopack reported a broken dev cache; clearing it for the next run." + clear_cache + fi exit "$status" } trap finish EXIT + # The apps boot while the runner installs Electron's libraries and bundles the shell. + sudo apt-get update -q + sudo apt-get install -yq xvfb libgtk-3-0t64 libnss3 libasound2t64 libgbm1 libxss1 \ + libxtst6 libatk-bridge2.0-0t64 libxkbcommon0 > /dev/null + # Bundle only: `bun run build` also fetches the macOS node-pty prebuilds for packaging, + # which a Linux run does not use. + (cd apps/desktop && bun run scripts/build.ts) started=$SECONDS + retried=0 until curl --fail --silent --max-time 10 http://127.0.0.1:3021/api/health > /dev/null && curl --fail --silent --max-time 10 http://127.0.0.1:3023/health > /dev/null; do - kill -0 "$server_pid" 2>/dev/null || { tail -n 200 "$report_dir/desktop-tools-next.log"; exit 1; } + if ! kill -0 "$server_pid" 2>/dev/null; then + tail -n 200 "$next_log" + if [ "$retried" = 0 ] && cache_broken; then + echo "::warning::Turbopack rejected the restored dev cache; restarting from an empty cache." + bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$server_tag" + clear_cache + : > "$next_log" + retried=1 + start_sim + continue + fi + exit 1 + fi kill -0 "$realtime_pid" 2>/dev/null || { tail -n 200 "$report_dir/desktop-tools-realtime.log"; exit 1; } [ $((SECONDS - started)) -lt "$READY_TIMEOUT_SECONDS" ] || { echo '::error::Local app did not become ready'; exit 1; } sleep 2 done + echo "Local apps ready $((SECONDS - started))s after the Electron bundle" cd apps/desktop SIM_DESKTOP_E2E_SIM_URL=http://127.0.0.1:3021 \ SIM_DESKTOP_E2E_PROXY_PORT=3020 \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be203bb1059..5f693d4e625 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,13 +18,11 @@ name: ci on: push: branches: [main, staging, dev] + # No path filter: `checks / ci` is the required status check, and a pull request whose run + # never starts never reports it, so it could not merge. Docs-only pull requests are rare enough + # that running CI on them costs nothing measurable. pull_request: branches: [main, staging, dev] - # Docs content and markdown don't affect the app build or images; push - # runs stay unfiltered because they feed the deploy pipeline. - paths-ignore: - - 'apps/docs/content/**' - - '**/*.md' concurrency: group: ci-${{ github.ref }} diff --git a/scripts/update-integration-durations.ts b/scripts/update-integration-durations.ts new file mode 100644 index 00000000000..32c9f876db9 --- /dev/null +++ b/scripts/update-integration-durations.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env bun +/** + * Rewrites `vitest.integration-durations.json`, the per-file weights the integration shards are + * balanced by (`DurationBalancedSequencer` in `vitest.shared.ts`), from a CI run's integration reports. + * + * gh run download -p 'integration-reports-*' -D /tmp/integration-reports + * bun run scripts/update-integration-durations.ts /tmp/integration-reports + * + * A file's weight is its wall time including the import and collection that precede its first + * test (the gap since the previous file in the same shard ended), averaged across the push and + * migrate provisioning runs. apps/sim runs its files one at a time, so those gaps are real shard + * time; where files overlap, a file counts only its own time. Weights only steer balance; a stale + * file never changes what runs. + */ +import { readdir, readFile, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { createLogger } from '@sim/logger' + +interface JsonReport { + testResults: { name: string; startTime: number; endTime: number }[] +} + +const logger = createLogger('UpdateIntegrationDurations') +const REPORT_NAME = 'integration.json' +const ROOT = path.resolve(import.meta.dir, '..') +const OUTPUT = path.join(ROOT, 'vitest.integration-durations.json') + +async function findReports(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true, recursive: true }) + return entries + .filter((entry) => entry.isFile() && entry.name === REPORT_NAME) + .map((entry) => path.join(entry.parentPath, entry.name)) +} + +/** `/home/runner/_work/sim/sim/apps/sim/lib/x.integration.ts` → `apps/sim/lib/x.integration.ts` */ +function repoPath(absolute: string): string | null { + const match = absolute.match(/\/((?:apps|packages)\/.+)$/) + return match ? match[1] : null +} + +const source = process.argv[2] +if (!source) { + logger.error('Usage: bun run scripts/update-integration-durations.ts ') + process.exit(1) +} + +const samples = new Map() +const reports = await findReports(path.resolve(source)) +for (const reportPath of reports) { + const report = JSON.parse(await readFile(reportPath, 'utf8')) as JsonReport + const files = [...report.testResults].sort((a, b) => a.startTime - b.startTime) + let previousEnd = Number.NEGATIVE_INFINITY + for (const file of files) { + const key = repoPath(file.name) + // The gap before a file is its import time only when nothing else was running: in a run + // with file parallelism (packages/db) files overlap, and a file then counts its own time. + const start = + previousEnd > Number.NEGATIVE_INFINITY && previousEnd <= file.startTime + ? previousEnd + : file.startTime + if (key) samples.set(key, [...(samples.get(key) ?? []), (file.endTime - start) / 1000]) + previousEnd = Math.max(previousEnd, file.endTime) + } +} + +if (samples.size === 0) { + logger.error(`No ${REPORT_NAME} reports with test results under ${source}`) + process.exit(1) +} + +const durations = Object.fromEntries( + [...samples.keys()].sort().map((key) => { + const values = samples.get(key) ?? [] + const mean = values.reduce((sum, value) => sum + value, 0) / values.length + return [key, Math.round(mean * 10) / 10] + }) +) +await writeFile(OUTPUT, `${JSON.stringify(durations, null, 2)}\n`) +logger.info(`Wrote ${samples.size} file weights from ${reports.length} reports to ${OUTPUT}`) diff --git a/vitest.integration-durations.json b/vitest.integration-durations.json new file mode 100644 index 00000000000..a4770d55a6c --- /dev/null +++ b/vitest.integration-durations.json @@ -0,0 +1,185 @@ +{ + "apps/sim/app/api/auth/oauth2/token/route.integration.ts": 3.9, + "apps/sim/app/api/billing/update-cost/route.integration.ts": 2.7, + "apps/sim/app/api/copilot/chat/stream/desktop-executor.integration.ts": 6.8, + "apps/sim/app/api/knowledge/connectors/member-sync/route.integration.ts": 1.8, + "apps/sim/app/api/v1/knowledge/route.integration.ts": 2.6, + "apps/sim/background/cleanup-table-row-ttl-predicate.integration.ts": 0.5, + "apps/sim/background/cleanup-table-row-ttl.integration.ts": 5.7, + "apps/sim/ee/access-requests/lib/application/flow.integration.ts": 1.1, + "apps/sim/ee/access-requests/lib/impact.integration.ts": 0.8, + "apps/sim/ee/access-requests/lib/repository.integration.ts": 0.4, + "apps/sim/ee/scim/lib/managed-membership.integration.ts": 0.6, + "apps/sim/ee/workspace-forking/application/revision.integration.ts": 2, + "apps/sim/ee/workspace-forking/lib/lineage/fork-lock-order.integration.ts": 1.7, + "apps/sim/executor/handlers/agent/memory-harness.integration.ts": 6.9, + "apps/sim/lib/auth/oauth-provider-lifecycle.integration.ts": 3.1, + "apps/sim/lib/auth/oauth-token-family.integration.ts": 1, + "apps/sim/lib/auth/sim-auth-adapter.integration.ts": 0.7, + "apps/sim/lib/auth/sso/application/admit-sso-user.integration.ts": 0.5, + "apps/sim/lib/auth/sso/primary-provider.integration.ts": 1.8, + "apps/sim/lib/billing/calculations/usage-reservation.integration.ts": 0.3, + "apps/sim/lib/billing/core/organization-activity.integration.ts": 0.8, + "apps/sim/lib/billing/core/organization-usage-pagination.integration.ts": 0.6, + "apps/sim/lib/billing/core/plan.integration.ts": 0.5, + "apps/sim/lib/billing/core/reporting-usage-cache.integration.ts": 0.9, + "apps/sim/lib/billing/core/usage-analytics-queries.integration.ts": 0.6, + "apps/sim/lib/billing/core/usage-log.integration.ts": 15.7, + "apps/sim/lib/billing/core/usage-threshold-email.integration.ts": 0.5, + "apps/sim/lib/billing/organizations/member-limits.integration.ts": 0.9, + "apps/sim/lib/core/async-jobs/backends/trigger-dev.integration.ts": 0.7, + "apps/sim/lib/core/outbox/retention.integration.ts": 0.5, + "apps/sim/lib/core/outbox/service.integration.ts": 7.2, + "apps/sim/lib/core/rate-limiter/provider-capacity.integration.ts": 0.8, + "apps/sim/lib/credential-groups/__integration__/hubspot-mcp.integration.ts": 2.4, + "apps/sim/lib/credential-groups/__integration__/organization-account-outbound.integration.ts": 1.4, + "apps/sim/lib/dashboards/repository.integration.ts": 0.4, + "apps/sim/lib/db/advisory-locks.integration.ts": 0.7, + "apps/sim/lib/desktop/application/activity.integration.ts": 0.8, + "apps/sim/lib/desktop/application/executor.integration.ts": 1.8, + "apps/sim/lib/desktop/application/import.integration.ts": 0.9, + "apps/sim/lib/desktop/executor/bound-turn.integration.ts": 85.6, + "apps/sim/lib/desktop/executor/desktop-call-classifier.integration.ts": 0.5, + "apps/sim/lib/desktop/source-request.integration.ts": 1.7, + "apps/sim/lib/environment/execution-environment.integration.ts": 0.6, + "apps/sim/lib/execution/remote-sandbox/session-file-provenance.integration.ts": 0.6, + "apps/sim/lib/execution/remote-sandbox/session-input-certification.integration.ts": 0.4, + "apps/sim/lib/knowledge/__integration__/application-acl.integration.ts": 3.9, + "apps/sim/lib/knowledge/__integration__/coda-live.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts": 2.1, + "apps/sim/lib/knowledge/__integration__/confluence-live.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/connector-deferral.integration.ts": 1.9, + "apps/sim/lib/knowledge/__integration__/connector-lease-pages.integration.ts": 25.3, + "apps/sim/lib/knowledge/__integration__/connector-lifecycle-locks.integration.ts": 2, + "apps/sim/lib/knowledge/__integration__/connector-partition-work.integration.ts": 1.7, + "apps/sim/lib/knowledge/__integration__/connector-permissions.integration.ts": 0.5, + "apps/sim/lib/knowledge/__integration__/connector-persistence-regressions.integration.ts": 3.7, + "apps/sim/lib/knowledge/__integration__/connector-save-concurrency.integration.ts": 2.3, + "apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts": 1.6, + "apps/sim/lib/knowledge/__integration__/directory-sync.integration.ts": 2.3, + "apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts": 1.3, + "apps/sim/lib/knowledge/__integration__/dormant-search-processing.integration.ts": 1.6, + "apps/sim/lib/knowledge/__integration__/embedding-insert-batches.integration.ts": 1.4, + "apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts": 3.2, + "apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts": 4.1, + "apps/sim/lib/knowledge/__integration__/external-file-provenance.integration.ts": 2, + "apps/sim/lib/knowledge/__integration__/filtered-search.integration.ts": 1.9, + "apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/gitlab-live.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/gitlab-workspace.integration.ts": 3.9, + "apps/sim/lib/knowledge/__integration__/google-drive-live.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/google-drive-member-live.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts": 3, + "apps/sim/lib/knowledge/__integration__/kb-block-search.integration.ts": 0.9, + "apps/sim/lib/knowledge/__integration__/knowledge-base-deletion.integration.ts": 0.8, + "apps/sim/lib/knowledge/__integration__/knowledge-base-list.integration.ts": 2.6, + "apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts": 0.9, + "apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts": 26.1, + "apps/sim/lib/knowledge/__integration__/member-document-lifecycle.integration.ts": 8.6, + "apps/sim/lib/knowledge/__integration__/member-scope-renewal.integration.ts": 1.6, + "apps/sim/lib/knowledge/__integration__/ocr-input-failures.integration.ts": 1.7, + "apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts": 1.5, + "apps/sim/lib/knowledge/__integration__/provider-admission.integration.ts": 0.8, + "apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts": 2.2, + "apps/sim/lib/knowledge/__integration__/providers-live.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/purged-detach-reservation.integration.ts": 5, + "apps/sim/lib/knowledge/__integration__/reranker-admission.integration.ts": 0.7, + "apps/sim/lib/knowledge/__integration__/scale.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts": 1.6, + "apps/sim/lib/knowledge/__integration__/search-latency.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts": 5.4, + "apps/sim/lib/knowledge/__integration__/search-reference-batching.integration.ts": 0.6, + "apps/sim/lib/knowledge/__integration__/search-source-pagination.integration.ts": 1.4, + "apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts": 3, + "apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts": 3, + "apps/sim/lib/knowledge/__integration__/slack-empty-threads.integration.ts": 2.1, + "apps/sim/lib/knowledge/__integration__/slack-live.integration.ts": 0, + "apps/sim/lib/knowledge/__integration__/slack-search-turns.integration.ts": 2.1, + "apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts": 3.1, + "apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts": 1.6, + "apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts": 5.1, + "apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts": 1.3, + "apps/sim/lib/knowledge/__integration__/user-document-visibility.integration.ts": 0.5, + "apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts": 1.8, + "apps/sim/lib/knowledge/__integration__/workspace-kb-document-access.integration.ts": 1.2, + "apps/sim/lib/knowledge/__integration__/workspace-lifecycle.integration.ts": 2.9, + "apps/sim/lib/knowledge/access/group-membership.integration.ts": 0.5, + "apps/sim/lib/knowledge/access/predicate.integration.ts": 16.5, + "apps/sim/lib/knowledge/application/github-installations.integration.ts": 0.8, + "apps/sim/lib/knowledge/application/search-index.integration.ts": 0.5, + "apps/sim/lib/knowledge/application/slack-search/assistant.integration.ts": 0.8, + "apps/sim/lib/knowledge/connectors/external-directory.integration.ts": 0.2, + "apps/sim/lib/knowledge/connectors/sync-content-pass.integration.ts": 1, + "apps/sim/lib/knowledge/connectors/sync-persistence.integration.ts": 0.6, + "apps/sim/lib/knowledge/search/history/history.integration.ts": 0.6, + "apps/sim/lib/knowledge/tags/date-tag-condition.integration.ts": 0.9, + "apps/sim/lib/logs/execution/completion-ledger-order.integration.ts": 2.6, + "apps/sim/lib/logs/execution/start-execution.integration.ts": 1.8, + "apps/sim/lib/memory/conversation-store.integration.ts": 2.8, + "apps/sim/lib/memory/message-provenance.integration.ts": 2.1, + "apps/sim/lib/memory/summary-store.integration.ts": 0.8, + "apps/sim/lib/mothership/agent-cli/workbench-file-provenance.integration.ts": 1.3, + "apps/sim/lib/mothership/async-runs/browser-download-claim.integration.ts": 0.6, + "apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts": 29.1, + "apps/sim/lib/mothership/async-runs/tool-identity.integration.ts": 3.2, + "apps/sim/lib/mothership/billing/service-store.integration.ts": 0.7, + "apps/sim/lib/mothership/chat/application/set-effort.integration.ts": 0.8, + "apps/sim/lib/mothership/chat/chat-mcp-servers.integration.ts": 0.6, + "apps/sim/lib/mothership/request/session/buffer-ttl.integration.ts": 12.3, + "apps/sim/lib/mothership/request/session/recovery-storm.integration.ts": 217.3, + "apps/sim/lib/mothership/request/session/replay-budget.integration.ts": 39, + "apps/sim/lib/mothership/request/session/replay-gap.integration.ts": 62.2, + "apps/sim/lib/mothership/request/session/stream-recovery.integration.ts": 6.9, + "apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts": 1, + "apps/sim/lib/mothership/tasks/application/prepare-wake.integration.ts": 12.3, + "apps/sim/lib/mothership/tools/client/desktop-tool-authorization.integration.ts": 6.8, + "apps/sim/lib/mothership/tools/client/desktop-tool-chat-view-lease.integration.ts": 6.9, + "apps/sim/lib/mothership/tools/client/desktop-tool-executor.integration.ts": 4.8, + "apps/sim/lib/mothership/tools/client/desktop-tool-pickup.integration.ts": 87.1, + "apps/sim/lib/projects/__integration__/foundation.integration.ts": 14.3, + "apps/sim/lib/selectors/__integration__/copilot-nested-selectors.integration.ts": 4.3, + "apps/sim/lib/sim-search/live/managed-mcp.integration.ts": 8.3, + "apps/sim/lib/sim-search/live/provider-discovery.integration.ts": 1.3, + "apps/sim/lib/table/analytics/postgres.integration.ts": 0.7, + "apps/sim/lib/table/row-changes.integration.ts": 0.8, + "apps/sim/lib/table/rows/row-writes.integration.ts": 12.5, + "apps/sim/lib/table/rows/secret-provenance.integration.ts": 1.9, + "apps/sim/lib/uploads/contexts/organization-logo/application.integration.ts": 0.8, + "apps/sim/lib/uploads/contexts/workspace/__integration__/file-names.integration.ts": 2.1, + "apps/sim/lib/uploads/contexts/workspace/__integration__/file-versions.integration.ts": 9.1, + "apps/sim/lib/uploads/upload-session/workspace-file-provenance.integration.ts": 0.6, + "apps/sim/lib/webhooks/registration-store.integration.ts": 0.9, + "apps/sim/lib/webhooks/trigger-block-deployment.integration.ts": 7.4, + "apps/sim/lib/workflows/application/compare-workflow-versions.integration.ts": 2.5, + "apps/sim/lib/workflows/application/read-workflow-version.integration.ts": 2.4, + "apps/sim/lib/workflows/custom-blocks/workspace-rows.integration.ts": 1.1, + "apps/sim/lib/workflows/executor/pause-persistence.integration.ts": 5.9, + "apps/sim/lib/workflows/orchestration/create-workflow-with-state.integration.ts": 3.1, + "apps/sim/lib/workspace-files/search/chunks.integration.ts": 13.9, + "apps/sim/lib/workspace-files/search/dispatcher.integration.ts": 14.5, + "apps/sim/lib/workspace-files/search/index-plan.integration.ts": 0.2, + "apps/sim/lib/workspaces/__integration__/fork-sync.integration.ts": 25, + "apps/sim/lib/workspaces/__integration__/http-cli.integration.ts": 8.6, + "apps/sim/lib/workspaces/__integration__/mapped-import.integration.ts": 6, + "apps/sim/lib/workspaces/__integration__/pagination.integration.ts": 0.8, + "apps/sim/lib/workspaces/__integration__/receipts.integration.ts": 0.6, + "apps/sim/lib/workspaces/organization-workspaces.integration.ts": 0.8, + "apps/sim/tools/powerbi/powerbi.integration.ts": 8.4, + "packages/db/embedding-width-migration.integration.ts": 0.4, + "packages/db/maintenance/search-retirement.integration.ts": 12, + "packages/db/member-sync-status-migration.integration.ts": 0, + "packages/db/organization-search-migration.integration.ts": 1.2, + "packages/db/permission-access-requests-migration.integration.ts": 0.1, + "packages/db/script-migrations/0013_backfill_legacy_knowledge_base_workspaces.integration.ts": 6, + "packages/db/script-migrations/0014_require_knowledge_base_owner.integration.ts": 0.2, + "packages/db/script-migrations/0016_backfill_search_vectors.integration.ts": 8, + "packages/db/script-migrations/0018_repair_workspace_file_content_revision.integration.ts": 0.1, + "packages/db/script-migrations/0019_tin_keyword_projection.integration.ts": 0.2, + "packages/db/script-migrations/0021_embedding_search_connector.integration.ts": 0.2, + "packages/db/script-migrations/0025_scope_keyword_projections.integration.ts": 0.4, + "packages/db/script-migrations/0027_retire_search_embeddings.integration.ts": 20.6, + "packages/db/scripts/connector-sync-schedule-precision.integration.ts": 0, + "packages/db/scripts/database-failure-classification.integration.ts": 1.1, + "packages/db/scripts/push.integration.ts": 7, + "packages/db/scripts/retired-columns.integration.ts": 0.3 +} diff --git a/vitest.shared.ts b/vitest.shared.ts index 31cdcdfd1e5..f5c24a401f9 100644 --- a/vitest.shared.ts +++ b/vitest.shared.ts @@ -1,4 +1,7 @@ +import { existsSync, readFileSync } from 'node:fs' +import path from 'node:path' import { configDefaults, defineConfig } from 'vitest/config' +import { BaseSequencer, type TestSpecification } from 'vitest/node' /** * The one Vitest base every workspace config extends with `mergeConfig`. @@ -22,6 +25,80 @@ export default defineConfig({ }, }) +/** Seconds a file of unknown duration is assumed to take: about the median measured file. */ +const UNKNOWN_FILE_SECONDS = 6 +/** Floor for any file: one that skips every test in CI still pays its import and collection. */ +const MIN_FILE_SECONDS = 1 +const DURATIONS_FILE = 'vitest.integration-durations.json' + +/** + * The committed weights, or none when the file is absent. A file that does not parse stops the run + * with the fix in the message rather than a bare SyntaxError. + */ +function readDurations(file: string): Record { + if (!existsSync(file)) return {} + try { + return JSON.parse(readFileSync(file, 'utf8')) + } catch (error) { + throw new Error( + `${DURATIONS_FILE} is not valid JSON. Regenerate it with scripts/update-integration-durations.ts.`, + { cause: error } + ) + } +} + +function findRepoRoot(from: string): string { + let dir = from + while (!existsSync(path.join(dir, 'bun.lock'))) { + const parent = path.dirname(dir) + if (parent === dir) throw new Error(`No bun.lock above ${from}`) + dir = parent + } + return dir +} + +/** + * Splits `--shard` runs by measured duration instead of by file count. + * + * Vitest's own shard() sorts files by path hash and hands each shard an equal number of them, so + * one long suite can land beside dozens of others and set the slowest shard. This packs files + * greedily, longest first, into the currently lightest shard, using the per-file seconds committed + * in `vitest.integration-durations.json` (refreshed by `scripts/update-integration-durations.ts`). + * + * Every shard computes the full partition from the same file list and the same committed weights, + * with ties broken by path and then by lowest shard index, so the shards are disjoint and together + * cover every file. A stale or missing weight only costs balance, never coverage. Ordering within a + * shard is Vitest's own sort(). + */ +class DurationBalancedSequencer extends BaseSequencer { + override async shard(files: TestSpecification[]): Promise { + const { index, count } = this.ctx.config.shard ?? { index: 1, count: 1 } + const root = findRepoRoot(this.ctx.config.root) + const durations = readDurations(path.join(root, DURATIONS_FILE)) + + const weighted = files.map((spec) => { + const key = path.relative(root, spec.moduleId).split(path.sep).join('/') + const recorded = durations[key] + const seconds = + typeof recorded === 'number' && Number.isFinite(recorded) ? recorded : UNKNOWN_FILE_SECONDS + return { spec, key, seconds: Math.max(seconds, MIN_FILE_SECONDS) } + }) + weighted.sort((a, b) => b.seconds - a.seconds || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)) + + const loads = new Array(count).fill(0) + const shards = Array.from({ length: count }, () => [] as TestSpecification[]) + for (const { spec, seconds } of weighted) { + let lightest = 0 + for (let shard = 1; shard < count; shard++) { + if (loads[shard] < loads[lightest]) lightest = shard + } + loads[lightest] += seconds + shards[lightest].push(spec) + } + return shards[index - 1] + } +} + /** * Overrides for a workspace's `--mode integration` run against real PostgreSQL and Redis. * @@ -38,5 +115,6 @@ export const integrationTestConfig = defineConfig({ hookTimeout: 30_000, reporters: ['default', 'json'], outputFile: { json: process.env.INTEGRATION_REPORT_PATH ?? 'test-results/integration.json' }, + sequence: { sequencer: DurationBalancedSequencer }, }, })