Copy extension binaries for CI tests

.github/workflows/job-compile-and-test.yml

@@ -45,53 +45,46 @@ jobs:
         run: yarn test
         working-directory: Extension
 
-      # These tests don't require the binary.
-      # On Linux, it is failing (before the tests actually run) with: Test run terminated with signal SIGSEGV.
-      # But it works on Linux during the E2E test.
-      - name: Run SingleRootProject tests
-        if: ${{ inputs.platform != 'linux' }}
-        run: yarn test --scenario=SingleRootProject --skipCheckBinaries
+      - name: Acquire Native Binaries
+        run: yarn install-and-copy-binaries-for-test
         working-directory: Extension
 
-      # NOTE : We can't run the test that require the native binary files
-      #        yet -- there will be an update soon that allows the tester to 
-      #        acquire them on-the-fly
-      # - name: Run languageServer integration tests
-      #   if: ${{ inputs.platform == 'windows' }}
-      #   run: yarn test --scenario=SingleRootProject
-      #   working-directory: Extension
+      - name: Run languageServer integration tests (Windows)
+        if: ${{ inputs.platform == 'windows' }}
+        run: yarn test --scenario=SingleRootProject
+        working-directory: Extension
 
-      # - name: Run E2E IntelliSense features tests
-      #   if: ${{ inputs.platform == 'windows' }}
-      #   run: yarn test --scenario=MultirootDeadlockTest
-      #   working-directory: Extension
+      - name: Run E2E IntelliSense features tests (Windows)
+        if: ${{ inputs.platform == 'windows' }}
+        run: yarn test --scenario=MultirootDeadlockTest
+        working-directory: Extension
 
-      # - name: Run E2E IntelliSense features tests
-      #   if: ${{ inputs.platform == 'windows' }}
-      #   run: yarn test --scenario=RunWithoutDebugging
-      #   working-directory: Extension
+      - name: Run RunWithoutDebugging tests (Windows)
+        if: ${{ inputs.platform == 'windows' }}
+        run: yarn test --scenario=RunWithoutDebugging
+        working-directory: Extension
 
       # NOTE: For mac/linux run the tests with xvfb-action for UI support.
       # Another way to start xvfb https://github.com/microsoft/vscode-test/blob/master/sample/azure-pipelines.yml
 
-      # - name: Run languageServer integration tests (xvfb)
-      #   if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }}
-      #   uses: coactions/setup-xvfb@v1
-      #   with:
-      #     run: yarn test --scenario=SingleRootProject
-      #     working-directory: Extension
-
-      # - name: Run E2E IntelliSense features tests (xvfb)
-      #   if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }}
-      #   uses: coactions/setup-xvfb@v1
-      #   with:
-      #     run: yarn test --scenario=MultirootDeadlockTest
-      #     working-directory: Extension
-
-      # - name: Run E2E IntelliSense features tests (xvfb)
-      #   if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }}
-      #   uses: coactions/setup-xvfb@v1
-      #   with:
-      #     run: yarn test --scenario=RunWithoutDebugging
-      #     working-directory: Extension
+      - name: Run languageServer integration tests (linux/macOS)
+        if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }}
+        uses: coactions/setup-xvfb@v1
+        with:
+          run: yarn test --scenario=SingleRootProject
+          working-directory: Extension
+
+      - name: Run E2E IntelliSense features tests (linux/macOS)
+        if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }}
+        uses: coactions/setup-xvfb@v1
+        with:
+          run: yarn test --scenario=MultirootDeadlockTest
+          working-directory: Extension
+
+      - name: Run RunWithoutDebugging tests (linux/macOS)
+        if: ${{ inputs.platform == 'mac' || inputs.platform == 'linux' }}
+        uses: coactions/setup-xvfb@v1
+        with:
+          run: yarn test --scenario=RunWithoutDebugging --scenario-arg=skipExternalConsole
+          working-directory: Extension
 

Extension/.scripts/common.ts

@@ -20,9 +20,18 @@ import { verbose } from '../src/Utility/Text/streams';
 export const $root = resolve(`${__dirname}/..`);
 export let $cmd = 'main';
 export let $scenario = '';
+export const $scenarioArgs: string[] = [];
 
 // loop through the args and pick out --scenario=... and remove it from the $args and set $scenario
 process.argv.slice(2).filter(each => !(each.startsWith('--scenario=') && ($scenario = each.substring('--scenario='.length))));
+// parse out the scenario arguments.
+process.argv.slice(2).reduce<string[]>((acc, arg) => {
+    if (arg.startsWith('--scenario-arg=')) {
+        acc.push(arg.substring('--scenario-arg='.length));
+    }
+    return acc;
+}, $scenarioArgs);
+
 export const $args = process.argv.slice(2).filter(each => !each.startsWith('--'));
 export const $switches = process.argv.slice(2).filter(each => each.startsWith('--'));
 

Extension/.scripts/copyExtensionBinaries.ts

@@ -74,16 +74,15 @@ async function getInstalledExtensions(root: string): Promise<InstalledExtension[
 }
 
 async function findLatestInstalledExtension(providedPath?: string): Promise<string> {
-    if (providedPath) {
-        return providedPath;
-    }
-
     const searchRoots: string[] = [
         join(homedir(), '.vscode', 'extensions'),
         join(homedir(), '.vscode-insiders', 'extensions'),
         join(homedir(), '.vscode-server', 'extensions'),
         join(homedir(), '.vscode-server-insiders', 'extensions')
     ];
+    if (providedPath) {
+        searchRoots.unshift(join(providedPath, 'extensions'));
+    }
 
     const installed: InstalledExtension[] = (await Promise.all(searchRoots.map(each => getInstalledExtensions(each)))).flat();
     if (!installed.length) {

Extension/.scripts/installAndCopyBinaries.ts

@@ -0,0 +1,24 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All Rights Reserved.
+ * See 'LICENSE' in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import { spawnSync } from 'child_process';
+import { verbose } from '../src/Utility/Text/streams';
+import { heading } from './common';
+import * as copy from './copyExtensionBinaries';
+import { install, isolated } from "./vscode";
+
+export async function main() {
+    console.log(heading(`Install VS Code`));
+    const { cli, args } = await install();
+
+    console.log(heading('Install latest C/C++ Extension'));
+    verbose(`Running command: ${cli} ${args.join(' ')} --install-extension ms-vscode.cpptools --pre-release`);
+    const result = spawnSync(cli, ['--install-extension', 'ms-vscode.cpptools', '--pre-release'], { encoding: 'utf-8', shell: true })
+    if (result.stdout) {
+        verbose(result.stdout.toString());
+    }
+
+    await copy.main(isolated);
+}

Extension/.scripts/test.ts

@@ -14,7 +14,7 @@ import { filepath } from '../src/Utility/Filesystem/filepath';
 import { is } from '../src/Utility/System/guards';
 import { verbose } from '../src/Utility/Text/streams';
 import { getTestInfo } from '../test/common/selectTests';
-import { $args, $root, $scenario, assertAnyFile, assertAnyFolder, brightGreen, checkBinaries, cmdSwitch, cyan, error, gray, green, readJson, red, writeJson } from './common';
+import { $args, $root, $scenario, $scenarioArgs, assertAnyFile, assertAnyFolder, brightGreen, checkBinaries, cmdSwitch, cyan, error, gray, green, readJson, red, writeJson } from './common';
 import { install, isolated, options } from './vscode';
 
 export { install, reset } from './vscode';
@@ -90,7 +90,8 @@ async function scenarioTests(assets: string, name: string, workspace: string) {
         extensionTestsPath: resolve($root, 'dist/test/common/selectTests'),
         launchArgs: workspace ? [...options.launchArgs, workspace] : options.launchArgs,
         extensionTestsEnv: {
-            SCENARIO: assets
+            SCENARIO: assets,
+            SCENARIO_ARGS: $scenarioArgs.join(',')
         }
     });
 }

Extension/package.json

@@ -6806,6 +6806,7 @@
         "generate-options-schema": "ts-node -T ./.scripts/generateOptionsSchema.ts",
         "copy-walkthrough-media": "ts-node -T ./.scripts/copyWalkthruMedia.ts",
         "copy-extension-binaries": "ts-node -T ./.scripts/copyExtensionBinaries.ts",
+        "install-and-copy-binaries-for-test": "ts-node -T ./.scripts/installAndCopyBinaries.ts",
         "translations-export": "yarn install && yarn prep && yarn generate-native-strings && gulp translations-export",
         "translations-generate": "gulp translations-generate",
         "translations-import": "gulp translations-import",

Extension/test/scenarios/RunWithoutDebugging/tests/runWithoutDebugging.terminals.test.ts

@@ -108,6 +108,7 @@ suite('Run Without Debugging Terminal and Arguments Test', function (this: Mocha
         'two words',
         path.join(workspacePath, 'input folder', 'three words.txt')
     ];
+    const skipExternalConsole = process.env.SCENARIO_ARGS?.includes('skipExternalConsole');
 
     suiteSetup(async function (): Promise<void> {
         const extension: vscode.Extension<any> = vscode.extensions.getExtension('ms-vscode.cpptools') || assert.fail('Extension not found');
@@ -170,6 +171,10 @@ suite('Run Without Debugging Terminal and Arguments Test', function (this: Mocha
             for (const profile of profiles) {
                 const profileSuffix = profile ? ` with ${profile} as the default terminal` : consoleCase.consoleMode === 'integratedTerminal' ? ' with default terminal' : '';
                 test(`No-debug launch via ${consoleCase.label} handles ${programCase.label}${profileSuffix}`, async () => {
+                    if (skipExternalConsole && consoleCase.consoleMode === 'externalTerminal') {
+                        console.log(`\tSkipping external terminal test for ${programCase.label}`);
+                        return;
+                    }
                     await setWindowsDefaultTerminalProfile(profile);
 
                     disposeTerminals(executablePaths);