script to fill out missing api functions

Herebyfile.mjs

@@ -729,6 +729,18 @@ export const generateAPI = task({
     run: runGenerateAPI,
 });
 
+async function runGenerateAPIDiff() {
+    await run("node", ["tools/scripts/gen/apiDiff.mts"]);
+}
+
+export const generateAPIDiff = task({
+    name: "generate:api-diff",
+    description: "Generates the API compatibility report.",
+    dependencies: [generateAST, generateAPI],
+    outputs: ["tsc/testdata/apiDiff.txt"],
+    run: runGenerateAPIDiff,
+});
+
 const vendorJsonrpcDir = "packages/typescript/vendor/vscode-jsonrpc";
 const vendorJsonrpcSrc = "node_modules/vscode-jsonrpc";
 // Files copied verbatim from the installed vscode-jsonrpc package into the
@@ -776,11 +788,12 @@ async function runGenerate() {
         },
         runGenerateVendor,
     ], 4);
+    await runGenerateAPIDiff();
 }
 
 export const generate = task({
     name: "generate",
-    description: "Runs all code generation, including AST, LSP, APIs, extension localization, and vendored dependencies.",
+    description: "Runs all code generation, including AST, LSP, APIs, the API compatibility report, extension localization, and vendored dependencies.",
     run: runGenerate,
 });
 

tools/scripts/gen/apiDiff.mts

@@ -0,0 +1,304 @@
+import fs from "node:fs";
+import path from "node:path";
+import ts from "typescript";
+import { repoRoot } from "./utils.mts";
+
+const oldApiPath = path.join(repoRoot, "node_modules", "typescript", "lib", "typescript.d.ts");
+const newApiPackageDirectory = path.join(repoRoot, "packages", "typescript");
+const newApiEntryPoints = [
+    "src/api/async/api.ts",
+    "src/api/fs.ts",
+    "src/api/proto.ts",
+    "src/ast/index.ts",
+    "src/ast/is.ts",
+    "src/ast/factory.generated.ts",
+    "src/ast/utils.ts",
+    "src/ast/scanner.ts",
+    "src/ast/visitor.ts",
+    "src/ast/clone.ts",
+].map(file => path.join(newApiPackageDirectory, file));
+const outputPath = path.join(repoRoot, "tsc", "testdata", "apiDiff.txt");
+const typeMappings = new Map<string, string | string[]>([
+    ["TypeChecker", "Checker"],
+    ["BuilderProgram", "Program"],
+    ["Program", "Program"],
+    ["SolutionBuilder", "BuildOrchestrator"],
+    ["Project", "Project"],
+    ["LanguageService", "Project"],
+    ["Scanner", "Scanner"],
+    ["Printer", "Printer"],
+    ["Signature", "Signature"],
+    ["SourceFile", "SourceFile"],
+    ["Symbol", "Symbol"],
+    ["Type", ["Type", "ts"]],
+    ["Node", "Node"],
+    ["NodeFactory", "ts"],
+    ["ts", ["ts", "API"]],
+]);
+
+const typescriptVersion = JSON.parse(
+    fs.readFileSync(path.join(repoRoot, "node_modules", "typescript", "package.json"), "utf8"),
+).version as string;
+const oldApiSourceFiles = [createSourceFile(oldApiPath)];
+const newApiProgram = createNewApiProgram();
+const newApiChecker = newApiProgram.getTypeChecker();
+const newApiExports = getEntryPointExports(newApiProgram, newApiChecker, newApiEntryPoints);
+
+const oldApiTypes = getTypesWithMethods(oldApiSourceFiles, visitOld);
+oldApiTypes.set("ts", getNamespaceFunctions(oldApiSourceFiles, "ts"));
+
+const newApiTypes = getExportedTypesWithMethods(newApiExports);
+newApiTypes.set("ts", [getExportedTopLevelFunctions(newApiExports, newApiChecker), new Set<string>()]);
+
+const acceptedDiffs = getAcceptedDiffs();
+
+const report: string[] = [
+    "# This report is generated by tools/scripts/gen/apidiff.mts.",
+    `# Old API: TypeScript ${typescriptVersion} (node_modules/typescript/lib/typescript.d.ts)`,
+    "# New API: public package entry points",
+];
+
+const comparisons = [...typeMappings].map(([oldType, newTypes]) => {
+    const oldMethods = oldApiTypes.get(oldType)!;
+    const acceptedChanges = acceptedDiffs.get(oldType) ?? new Map<string, string>();
+    let missing: string[] = [...oldMethods.keys()];
+    const present: string[] = [];
+    const newTypeArray = Array.isArray(newTypes) ? newTypes : [newTypes];
+    for (const newType of newTypeArray) {
+        const currMissing: string[] = [];
+        const newMethods = newApiTypes.get(newType)!;
+        for (const method of missing) {
+            if (newMethods[0].has(method)) {
+                present.push(method);
+                newMethods[0].delete(method);
+                newMethods[1].add(method);
+            }
+            else if (newMethods[1].has(method)) {
+                present.push(method);
+            }
+            else if (acceptedChanges.has(method)) {
+                present.push(method);
+                const [newType, newMethodName] = acceptedChanges.get(method)!.split(".");
+                const actualNewType = newApiTypes.get(newType);
+                if (actualNewType) {
+                    actualNewType[0].delete(newMethodName);
+                    actualNewType[1].add(newMethodName);
+                }
+            }
+            else {
+                currMissing.push(method);
+            }
+        }
+        missing = currMissing;
+    }
+    return {
+        oldType,
+        newTypes,
+        present: present.sort(),
+        missing: missing.sort(),
+    };
+});
+
+report.push(
+    `Types not compared:`,
+);
+for (const [type, _] of oldApiTypes) {
+    typeMappings.has(type) || report.push(`- ${type}`);
+}
+report.push(
+    "",
+);
+
+for (const comparison of comparisons) {
+    const { oldType, newTypes, present, missing } = comparison;
+    const newType = Array.isArray(newTypes) ? newTypes.join(", ") : newTypes;
+    report.push(
+        "",
+        `# ${oldType} (old) methods compared to ${newType} (new).`,
+        `## (${present.length}) ${oldType} methods matched in async ${newType}  ##`,
+        "",
+        `## ${oldType} methods missing from async ${newType} (${missing.length}) ##`,
+        ...missing,
+        "",
+    );
+}
+
+report.push("\n\n # New Methods in async API (temporarily reported for increased ease of matching)");
+
+for (const [newType, newMethods] of newApiTypes) {
+    report.push(
+        `## New ${newType} methods in async API (${newMethods[0].size}) ##`,
+        ...newMethods[0],
+        "",
+    );
+}
+
+fs.writeFileSync(outputPath, report.join("\n"));
+
+function createSourceFile(file: string): ts.SourceFile {
+    return ts.createSourceFile(
+        file,
+        fs.readFileSync(file, "utf8"),
+        ts.ScriptTarget.Latest,
+        true,
+    );
+}
+
+function createNewApiProgram(): ts.Program {
+    const configPath = path.join(newApiPackageDirectory, "tsconfig.json");
+    const configFile = ts.readConfigFile(configPath, ts.sys.readFile);
+    if (configFile.error) {
+        throw new Error(ts.flattenDiagnosticMessageText(configFile.error.messageText, "\n"));
+    }
+    const config = ts.parseJsonConfigFileContent(configFile.config, ts.sys, newApiPackageDirectory, { noEmit: true }, configPath);
+    if (config.errors.length > 0) {
+        throw new Error(config.errors.map(error => ts.flattenDiagnosticMessageText(error.messageText, "\n")).join("\n"));
+    }
+    return ts.createProgram({
+        rootNames: config.fileNames,
+        options: config.options,
+        projectReferences: config.projectReferences,
+    });
+}
+
+type EntryPointExports = ReadonlyMap<string, ReadonlySet<ts.Symbol>>;
+
+function getEntryPointExports(program: ts.Program, checker: ts.TypeChecker, entryPoints: readonly string[]): EntryPointExports {
+    const exports = new Map<string, Set<ts.Symbol>>();
+    for (const entryPoint of entryPoints) {
+        const sourceFile = program.getSourceFile(entryPoint);
+        if (!sourceFile) throw new Error(`Could not find API entry point ${entryPoint}`);
+        const moduleSymbol = checker.getSymbolAtLocation(sourceFile);
+        if (!moduleSymbol) throw new Error(`Could not resolve API entry point ${entryPoint}`);
+        for (const exportedSymbol of checker.getExportsOfModule(moduleSymbol)) {
+            const symbols = exports.get(exportedSymbol.name) ?? new Set<ts.Symbol>();
+            symbols.add(resolveAlias(exportedSymbol, checker));
+            exports.set(exportedSymbol.name, symbols);
+        }
+    }
+    return exports;
+}
+
+function resolveAlias(symbol: ts.Symbol, checker: ts.TypeChecker): ts.Symbol {
+    return symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol;
+}
+
+function getTypesWithMethods<T>(sourceFiles: readonly ts.SourceFile[], visitNode: (node: ts.Node, types: Map<string, T>) => void): Map<string, T> {
+    const types = new Map<string, T>();
+    for (const sourceFile of sourceFiles) {
+        visit(sourceFile);
+    }
+    return types;
+
+    function visit(node: ts.Node): void {
+        visitNode(node, types);
+        ts.forEachChild(node, visit);
+    }
+}
+
+function getAcceptedDiffs(): Map<string, Map<string, string>> {
+    const diffs = new Map<string, Map<string, string>>();
+    const contents = fs.readFileSync(path.join(repoRoot, "tsc", "testdata", "apiAccepted.txt"), "utf-8").split("\n");
+    contents.slice(2).forEach(line => {
+        const [oldMethod, newMethod, _] = line.split("|");
+        const [oldType, oldMethodName] = oldMethod.split(".");
+        if (!diffs.has(oldType)) diffs.set(oldType, new Map<string, string>());
+        diffs.get(oldType)!.set(oldMethodName, newMethod);
+    });
+    return diffs;
+}
+
+function visitOld(node: ts.Node, types: Map<string, Set<string>>): void {
+    if ((ts.isClassDeclaration(node) || ts.isInterfaceDeclaration(node)) && node.name) {
+        const methods = types.get(node.name.text) ?? new Set<string>();
+        for (const member of node.members) {
+            if (
+                (ts.isMethodDeclaration(member) || ts.isMethodSignature(member))
+                && !hasModifier(member, ts.SyntaxKind.PrivateKeyword)
+                && !hasModifier(member, ts.SyntaxKind.ProtectedKeyword)
+            ) {
+                const name = getPropertyName(member.name);
+                if (name !== undefined) methods.add(name);
+            }
+        }
+        if (methods.size > 0) types.set(node.name.text, methods);
+    }
+}
+
+function getExportedTypesWithMethods(exports: EntryPointExports): Map<string, [notSeen: Set<string>, seen: Set<string>]> {
+    const types = new Map<string, [notSeen: Set<string>, seen: Set<string>]>();
+    for (const [exportName, symbols] of exports) {
+        const methods = new Set<string>();
+        for (const symbol of symbols) {
+            for (const declaration of symbol.declarations ?? []) {
+                if (ts.isClassDeclaration(declaration) || ts.isInterfaceDeclaration(declaration)) {
+                    addPublicMethods(declaration.members, methods);
+                }
+            }
+        }
+        if (methods.size > 0) types.set(exportName, [methods, new Set<string>()]);
+    }
+    return types;
+}
+
+function addPublicMethods(members: ts.NodeArray<ts.ClassElement | ts.TypeElement>, methods: Set<string>): void {
+    for (const member of members) {
+        if (
+            (ts.isMethodDeclaration(member) || ts.isMethodSignature(member))
+            && !hasModifier(member, ts.SyntaxKind.PrivateKeyword)
+            && !hasModifier(member, ts.SyntaxKind.ProtectedKeyword)
+        ) {
+            const name = getPropertyName(member.name);
+            if (name !== undefined) methods.add(name);
+        }
+    }
+}
+
+function getNamespaceFunctions(sourceFiles: readonly ts.SourceFile[], namespace: string): Set<string> {
+    const functions = new Set<string>();
+    for (const sourceFile of sourceFiles) {
+        for (const statement of sourceFile.statements) {
+            if (
+                ts.isModuleDeclaration(statement)
+                && statement.name.text === namespace
+                && statement.body
+                && ts.isModuleBlock(statement.body)
+            ) {
+                for (const member of statement.body.statements) {
+                    if (ts.isFunctionDeclaration(member) && member.name) {
+                        functions.add(member.name.text);
+                    }
+                }
+            }
+        }
+    }
+    return functions;
+}
+
+function getExportedTopLevelFunctions(exports: EntryPointExports, checker: ts.TypeChecker): Set<string> {
+    const functions = new Set<string>();
+    for (const [exportName, symbols] of exports) {
+        for (const symbol of symbols) {
+            const declaration = symbol.valueDeclaration;
+            if (
+                declaration
+                && checker.getTypeOfSymbolAtLocation(symbol, declaration).getCallSignatures().length > 0
+            ) {
+                functions.add(exportName);
+                break;
+            }
+        }
+    }
+    return functions;
+}
+
+function hasModifier(node: ts.Node, kind: ts.SyntaxKind): boolean {
+    return ts.canHaveModifiers(node) && ts.getModifiers(node)?.some(modifier => modifier.kind === kind) === true;
+}
+
+function getPropertyName(name: ts.PropertyName): string | undefined {
+    if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) {
+        return name.text;
+    }
+    return undefined;
+}