From 4b64b3597f3e590460aeb70d1a69e325bfcc6177 Mon Sep 17 00:00:00 2001 From: Cal Barkman Date: Fri, 26 Jun 2026 20:49:17 -0700 Subject: [PATCH] feat(parser): add pipeline parser and CLI commands --- package.json | 71 +- src/extension.ts | 4 + src/parsers/pipelineParser.ts | 646 ++++++++++++++++++ src/providers/pipelineParserCommand.ts | 228 +++++++ .../pipelineParserCommandRegistrations.ts | 127 ++++ src/services/cache/componentCacheManager.ts | 31 + src/services/component/componentService.ts | 139 ++++ src/utils/gitlabVariables.ts | 628 +++++++++-------- src/utils/httpClient.ts | 20 +- tests/unit/pipelineParser.test.js | 361 ++++++++++ tests/unit/url-parsing.test.js | 123 ++++ 11 files changed, 2107 insertions(+), 271 deletions(-) create mode 100644 src/parsers/pipelineParser.ts create mode 100644 src/providers/pipelineParserCommand.ts create mode 100644 src/providers/pipelineParserCommandRegistrations.ts create mode 100644 tests/unit/pipelineParser.test.js create mode 100644 tests/unit/url-parsing.test.js diff --git a/package.json b/package.json index 722e0722..fa41de56 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,44 @@ "gitlabComponentHelper.cacheTime": { "type": "number", "default": 3600, - "description": "Cache time for components in seconds" + "description": "How long to keep components cached in seconds. Default is 1 hour.", + "scope": "resource" + }, + "gitlabComponentHelper.parser.preferLocalIncludes": { + "type": "boolean", + "default": true, + "markdownDescription": "When enabled, `project:` and `component:` includes will first check your local workspace for the corresponding files and use them if they exist. This is useful for testing a local copy of a PEP or shared configuration. Disable this to strictly fetch them from the remote GitLab server.", + "scope": "resource" + }, + "gitlabComponentHelper.parser.activePolicyOverride": { + "type": "string", + "default": "", + "markdownDescription": "The name of a specific Pipeline Execution Policy (PEP) to isolate and parse. If set, only this policy will be parsed and injected. You can select this via the `GitLab CI: Select Active Policy Override` command.", + "scope": "resource" + }, + "gitlabComponentHelper.projectPath": { + "type": "string", + "default": "", + "description": "The GitLab project path for the current workspace (e.g., 'my-group/my-project'). Required for resolving project-relative includes in the parser.", + "scope": "resource" + }, + "gitlabComponentHelper.parser.pepProjectPathOverride": { + "type": "string", + "default": "", + "markdownDescription": "Explicitly set the path to your Security Policy Project (PEP) to bypass GraphQL auto-discovery. Useful for Instance-level policies (e.g. `compliance-group/policy-repo`).", + "scope": "resource" + }, + "gitlabComponentHelper.trustedIncludeRoot": { + "type": [ + "string", + "array" + ], + "items": { + "type": "string" + }, + "default": [], + "markdownDescription": "Additional local directories that are trusted for `local:` includes in the parser. By default, only the current workspace folders are trusted. Use this if your Pipeline Execution Policy (PEP) or other includes reside in a folder outside your primary workspace.", + "scope": "resource" }, "gitlabComponentHelper.logLevel": { "type": "string", @@ -228,6 +265,31 @@ "command": "gitlab-component-helper.browseComponents", "title": "GitLab CI: Browse Components" }, + { + "command": "gitlab-component-helper.parsePipeline", + "title": "GitLab CI: Parse Pipeline (Text Output)", + "icon": "$(list-flat)" + }, + { + "command": "gitlab-component-helper.selectPolicyOverride", + "title": "GitLab CI: Select Active Policy Override", + "category": "GitLab CI" + }, + { + "command": "gitlab-component-helper.setProjectPath", + "title": "GitLab CI: Set Active Project Path", + "category": "GitLab CI" + }, + { + "command": "gitlab-component-helper.selectLocalPepOverride", + "title": "GitLab CI: Select Local PEP File Override", + "category": "GitLab CI" + }, + { + "command": "gitlab-component-helper.clearPepOverrides", + "title": "GitLab CI: Clear PEP Overrides", + "category": "GitLab CI" + }, { "command": "gitlab-component-helper.refreshComponents", "title": "GitLab CI: Refresh Components Cache" @@ -264,9 +326,14 @@ "menus": { "editor/context": [ { - "when": "gitlabComponentHelper.isCiFile", + "when": "resourceFilename =~ /gitlab-ci\\.ya?ml$/", "command": "gitlab-component-helper.browseComponents", "group": "navigation" + }, + { + "when": "resourceFilename =~ /gitlab-ci\\.ya?ml$/", + "command": "gitlab-component-helper.parsePipeline", + "group": "navigation@2" } ] } diff --git a/src/extension.ts b/src/extension.ts index 3fff5bd5..6736ea5e 100755 --- a/src/extension.ts +++ b/src/extension.ts @@ -12,6 +12,7 @@ import { ValidationProvider } from './providers/validationProvider'; import type { CachedComponent } from './types/cache'; import type { GitLabYamlFragment } from './types/gitlab-catalog'; import type { HoverContext } from './providers/hoverContentBuilder'; +import { registerPipelineParserCommands } from './providers/pipelineParserCommandRegistrations'; /** Component payload passed to the `detachHover` command. Adds the hover-builder's location context. */ type DetachableComponent = Component & { _hoverContext?: HoverContext }; @@ -163,6 +164,9 @@ export function activate(context: vscode.ExtensionContext) { }) ); + // Register Pipeline Parser commands (TUI output) + registerPipelineParserCommands(context); + // Register command to refresh component cache logger.debug('[Extension] Registering refreshComponents command...', 'Extension'); context.subscriptions.push( diff --git a/src/parsers/pipelineParser.ts b/src/parsers/pipelineParser.ts new file mode 100644 index 00000000..e353f1ab --- /dev/null +++ b/src/parsers/pipelineParser.ts @@ -0,0 +1,646 @@ +import { parseYaml } from '../utils/yamlParser'; +import { getComponentService } from '../services/component/componentService'; +import { getComponentCacheManager } from '../services/cache/componentCacheManager'; +import * as vscode from 'vscode'; +import * as fs from 'fs'; +import * as path from 'path'; +import { expandComponentUrl, expandGitLabVariables } from '../utils/gitlabVariables'; + +export interface PipelineJob { + name: string; + stage: string; + source: string; +} + +export interface PipelineStage { + name: string; + jobs: PipelineJob[]; + isImplicit: boolean; +} + +export interface IncludeNode { + name: string; + children: IncludeNode[]; +} + +export interface PipelineGraph { + stages: PipelineStage[]; + includedSources: string[]; + includeTree: IncludeNode; + errors: string[]; +} + +export interface ComponentOrigin { + gitlabInstance: string; + projectPath: string; + ref: string; +} + +export interface ParserContext { + gitlabInstance?: string; + projectPath?: string; + customVariables?: Record; + activePolicyOverride?: string; + serverUrl?: string; + [key: string]: any; +} + +export interface IncludeDirective { + local?: string; + file?: string | string[]; + project?: string; + ref?: string; + remote?: string; + component?: string; + template?: string; +} + +const DEFAULT_STAGES = ['.pre', 'build', 'test', 'deploy', '.post']; +const RESERVED_KEYWORDS = new Set([ + 'image', 'services', 'stages', 'types', 'before_script', 'after_script', + 'variables', 'cache', 'include', 'pages', 'workflow', 'default', 'spec', + 'pipeline_execution_policy', 'content' +]); + +export class PipelineParser { + private maxDepth: number; + private visitedSources = new Set(); + private allJobs: PipelineJob[] = []; + private customStages: string[] = []; + private includedSources: string[] = []; + private includeTree: IncludeNode = { name: 'root', children: [] }; + private errors: string[] = []; + private allowedRoots: string[] | null = null; + private extraAllowedRoots: string[] = []; + private entryDirectory: string | null = null; + + constructor(maxDepth: number = 10) { + this.maxDepth = maxDepth; + } + + public async parse(content: string, sourceName: string, context?: ParserContext, extraIncludes?: IncludeDirective[]): Promise { + this.visitedSources.clear(); + this.allJobs = []; + this.customStages = []; + this.includedSources = [sourceName]; + this.includeTree = { name: sourceName, children: [] }; + this.errors = []; + this.allowedRoots = null; // Clear cached allowed roots for the new parsing run + this.extraAllowedRoots = []; + + if (path.isAbsolute(sourceName)) { + this.entryDirectory = path.dirname(sourceName); + } else { + this.entryDirectory = null; + } + + // Resolve any always-include entries first, before the main file. + if (extraIncludes && extraIncludes.length > 0) { + for (const inc of extraIncludes) { + if (inc.local && path.isAbsolute(inc.local)) { + this.extraAllowedRoots.push(path.dirname(inc.local)); + } + } + for (const inc of extraIncludes) { + await this.resolveInclude(inc, sourceName, 1, this.includeTree, context); + } + } + + await this.parseRecursive(content, sourceName, 0, this.includeTree, context); + + return this.buildGraph(); + } + + private async parseRecursive(content: string, sourceName: string, depth: number, parentNode: IncludeNode, context?: ParserContext, componentOrigin?: ComponentOrigin) { + if (depth >= this.maxDepth) { + this.errors.push(`Max recursion depth (${this.maxDepth}) reached at ${sourceName}`); + return; + } + + if (this.visitedSources.has(sourceName)) { + // Circular dependency or already visited + return; + } + this.visitedSources.add(sourceName); + + // Strip out the `spec:` block and split by --- to handle components + const parts = content.split(/^---\s*$/m); + let ciContent = content; + if (parts.length > 1) { + // Usually spec is before ---, and jobs are after. + ciContent = parts.slice(1).join('\n'); + } + + const parsed = parseYaml(ciContent); + if (!parsed || typeof parsed !== 'object') { + this.errors.push(`Failed to parse YAML for ${sourceName}`); + return; + } + + const parsedObj = parsed as Record; + if (parsedObj.stages && Array.isArray(parsedObj.stages)) { + for (const stage of parsedObj.stages) { + if (!this.customStages.includes(stage)) { + this.customStages.push(stage); + } + } + } + + // 2. Extract jobs + for (const key of Object.keys(parsedObj)) { + if (RESERVED_KEYWORDS.has(key) || key.startsWith('.')) { + // Skip reserved keywords and hidden jobs/anchors + continue; + } + + const jobObj = parsedObj[key]; + if (jobObj && typeof jobObj === 'object') { + const stage = (jobObj as Record).stage || 'test'; // default stage is test in GitLab CI + this.allJobs.push({ + name: key, + stage: stage, + source: sourceName + }); + } + } + + // 3. Extract includes + if (parsedObj.include) { + const includes = Array.isArray(parsedObj.include) ? parsedObj.include : [parsedObj.include]; + for (const inc of includes) { + await this.resolveInclude(inc, sourceName, depth + 1, parentNode, context, componentOrigin); + } + } + + // 4. Handle GitLab Pipeline Execution Policies (PEP). + // A PEP file has a top-level `pipeline_execution_policy` array. Each entry + // has a `pipeline` key which is an embedded CI document with its own + // stages, jobs, and includes. We extract them all here so they appear in + // the visualizer alongside the rest of the pipeline. + if (parsedObj.pipeline_execution_policy && Array.isArray(parsedObj.pipeline_execution_policy)) { + for (const policy of parsedObj.pipeline_execution_policy) { + if (!policy) continue; + + if (context?.activePolicyOverride && policy.name !== context.activePolicyOverride) { + continue; // Skip policies that don't match the override + } + + // GitLab PEPs use 'content' for the embedded pipeline, but we support + // 'pipeline' as a fallback (used in some documentation/earlier versions). + const pipelineDoc = policy.content || policy.pipeline; + if (!pipelineDoc || typeof pipelineDoc !== 'object') { + continue; + } + const policyLabel = policy.name + ? `PEP: ${policy.name} (${sourceName})` + : `PEP (${sourceName})`; + + // Register the policy so it appears in the "Included Sources" panel + if (!this.includedSources.includes(policyLabel)) { + this.includedSources.push(policyLabel); + } + + const pipelineObj = pipelineDoc as Record; + // Extract stages declared inside the policy pipeline + if (pipelineObj.stages && Array.isArray(pipelineObj.stages)) { + for (const stage of pipelineObj.stages) { + if (!this.customStages.includes(stage)) { + this.customStages.push(stage); + } + } + } + + // Extract inline jobs from the policy pipeline + for (const key of Object.keys(pipelineObj)) { + if (RESERVED_KEYWORDS.has(key) || key.startsWith('.') || key === 'stages') { + continue; + } + const jobObj = pipelineObj[key]; + if (jobObj && typeof jobObj === 'object') { + const stage = (jobObj as Record).stage || 'test'; + this.allJobs.push({ name: key, stage, source: policyLabel }); + } + } + + // Resolve includes declared inside the policy pipeline + if (pipelineObj.include) { + const pepIncludes = Array.isArray(pipelineObj.include) ? pipelineObj.include : [pipelineObj.include]; + for (const inc of pepIncludes) { + await this.resolveInclude(inc, sourceName, depth + 1, parentNode, context, componentOrigin); + } + } + } + } + } + + private async resolveInclude(inc: IncludeDirective | string, currentSource: string, depth: number, parentNode: IncludeNode, context?: ParserContext, componentOrigin?: ComponentOrigin) { + if (!inc) return; + + let targetUrl = ''; + let targetName = ''; + + try { + if (typeof inc === 'string') { + if (inc.startsWith('http')) { + targetUrl = inc; + targetName = inc; + } else if (!path.isAbsolute(inc) && inc.includes('@')) { + // String shorthand for component: 'group/project/component@1.0.0' + inc = { component: inc }; + } else if (!path.isAbsolute(inc) && inc.includes(':')) { + // String shorthand for project: 'group/project:file.yml' + const [project, file] = inc.split(':'); + inc = { project, file }; + } else { + await this.resolveLocalInclude(inc, currentSource, depth, parentNode, context, componentOrigin); + return; + } + } + + const directive = inc as IncludeDirective; + + if (directive.local) { + // Local include + await this.resolveLocalInclude(directive.local, currentSource, depth, parentNode, context, componentOrigin); + return; + } else if (directive.component) { + // Component include + // e.g., gitlab.com/my-group/my-project/my-component@1.0.0 + let componentUrl = directive.component; + + // Expand variables like $CI_SERVER_FQDN + componentUrl = expandComponentUrl(componentUrl, { + gitlabInstance: context?.gitlabInstance || 'gitlab.com', + serverUrl: context?.serverUrl, + projectPath: context?.projectPath, + customVariables: context?.customVariables + }); + // Strip protocol because logic expects domain/path + componentUrl = componentUrl.replace(/^https?:\/\//, ''); + + targetName = `component:${componentUrl}`; + + // Parse component URL to fetch the raw template + const componentService = getComponentService(); + const parsedUrl = componentService.parseCustomComponentUrl(`https://${componentUrl}`); + if (!parsedUrl) { + this.errors.push(`Could not parse component URL ${componentUrl}`); + return; + } + + let version = parsedUrl.version || 'main'; + if (version === '[current-branch-or-sha]') { + version = 'HEAD'; + this.errors.push(`Replaced missing variable $CI_COMMIT_SHA with HEAD for component ${directive.component}. Click here to set custom variables [action:openCustomVariables]`); + } + + const combinations = [ + `templates/${parsedUrl.name}/template.yml`, + `templates/${parsedUrl.name}.yml`, + `templates/template.yml` + ]; + + // Local Redirect: Try resolving locally first if configured to allow local overrides. + let resolvedLocally = false; + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + const preferLocal = config.get('visualizer.preferLocalIncludes', true); + + if (preferLocal) { + for (const templatePath of combinations) { + const resolved = await this.tryResolveLocal(templatePath, currentSource, context, parsedUrl.path); + if (resolved) { + const nodeName = `${targetName} (Local Override: ${resolved.path})`; + if (!this.includedSources.includes(nodeName)) { + this.includedSources.push(nodeName); + } + const node = { name: nodeName, children: [] }; + parentNode.children.push(node); + await this.parseRecursive(resolved.content, resolved.path, depth, node, context); + resolvedLocally = true; + break; + } + } + } + if (resolvedLocally) { + return; + } + + let fetched = false; + const cacheManager = getComponentCacheManager(); + + for (const templatePath of combinations) { + try { + const content = await cacheManager.fetchAndCacheRawTemplate(parsedUrl.gitlabInstance, parsedUrl.path, templatePath, version); + if (content && typeof content === 'string') { + this.includedSources.push(targetName); + const origin: ComponentOrigin = { + gitlabInstance: parsedUrl.gitlabInstance, + projectPath: parsedUrl.path, + ref: version + }; + const node = { name: targetName, children: [] }; + parentNode.children.push(node); + await this.parseRecursive(content, targetName, depth, node, context, origin); + fetched = true; + break; + } + } catch (e) { + // Continue trying next combination + } + } + + if (!fetched) { + this.errors.push(`Could not fetch component ${componentUrl}`); + } + return; + } else if (directive.project && directive.file) { + // Project include + let projectPath = directive.project; + projectPath = expandGitLabVariables(projectPath, { + gitlabInstance: context?.gitlabInstance || 'gitlab.com', + projectPath: context?.projectPath, + customVariables: context?.customVariables + }); + + const files = Array.isArray(directive.file) ? directive.file : [directive.file]; + const gitlabInstance = context?.gitlabInstance || 'gitlab.com'; + const componentService = getComponentService(); + + for (const file of files) { + let expandedFile = expandGitLabVariables(typeof file === 'string' ? file : String(file), { + gitlabInstance: context?.gitlabInstance || 'gitlab.com', + projectPath: context?.projectPath, + customVariables: context?.customVariables + }); + + targetName = `project:${projectPath}:${expandedFile}`; + const ref = directive.ref || 'HEAD'; + const cleanFile = expandedFile.replace(/^\//, ''); + + // Local Redirect: Try resolving locally first if configured. + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + const preferLocal = config.get('visualizer.preferLocalIncludes', true); + + if (preferLocal) { + const resolved = await this.tryResolveLocal(cleanFile, currentSource, context, projectPath); + if (resolved) { + const nodeName = `${targetName} (Local Override: ${resolved.path})`; + if (!this.includedSources.includes(nodeName)) { + this.includedSources.push(nodeName); + } + const node = { name: nodeName, children: [] }; + parentNode.children.push(node); + await this.parseRecursive(resolved.content, resolved.path, depth, node, context); + continue; // Move to next file in directive.file array + } + } + + try { + const cacheManager = getComponentCacheManager(); + const content = await cacheManager.fetchAndCacheRawTemplate(gitlabInstance, projectPath, cleanFile, ref); + if (content && typeof content === 'string') { + if (!this.includedSources.includes(targetName)) { + this.includedSources.push(targetName); + } + // Pass the project origin so local: includes inside this file resolve via GitLab API + const origin: ComponentOrigin = { gitlabInstance, projectPath, ref }; + const node = { name: targetName, children: [] }; + parentNode.children.push(node); + await this.parseRecursive(content, targetName, depth, node, context, origin); + } else { + this.errors.push(`Could not fetch project file ${directive.project}/${file}. Permission denied or file not found. If this is a restricted PEP file, please provide a local copy and enable the 'visualizer.preferLocalIncludes' setting.`); + } + } catch (e) { + this.errors.push(`Failed to fetch project file ${directive.project}/${file}: ${this.formatError(e)}. If this is a restricted PEP file, please provide a local copy and enable the 'visualizer.preferLocalIncludes' setting.`); + } + } + return; + } else if (directive.remote) { + // Remote include + let remoteUrl = directive.remote; + remoteUrl = expandGitLabVariables(remoteUrl, { + gitlabInstance: context?.gitlabInstance || 'gitlab.com', + serverUrl: context?.serverUrl, + projectPath: context?.projectPath, + customVariables: context?.customVariables + }); + targetUrl = remoteUrl; + targetName = `remote:${remoteUrl}`; + } else { + return; + } + + if (targetUrl) { + if (!this.includedSources.includes(targetName)) { + this.includedSources.push(targetName); + } + const service = getComponentService(); + const content = await service.httpClient.fetchText(targetUrl); + if (content) { + const node = { name: targetName, children: [] }; + parentNode.children.push(node); + await this.parseRecursive(content, targetName, depth, node, context); + } + } + } catch (e) { + this.errors.push(`Failed to resolve include ${targetName}: ${this.formatError(e)}`); + } + } + + private formatError(e: unknown): string { + return e instanceof Error ? e.message : String(e); + } + + private getAllowedRoots(): string[] { + if (this.allowedRoots !== null) { + return this.allowedRoots; + } + + const workspaceFolders = vscode.workspace.workspaceFolders; + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + const trustedRootsConfig = config.get('trustedIncludeRoot', []); + const extraRoots = Array.isArray(trustedRootsConfig) ? trustedRootsConfig : [trustedRootsConfig]; + + const rawRoots = [ + ...(workspaceFolders?.map(f => f.uri.fsPath) || []), + ...(this.entryDirectory ? [this.entryDirectory] : []), + ...extraRoots.filter(r => r && typeof r === 'string').map(r => path.resolve(r)), + ...this.extraAllowedRoots + ]; + + this.allowedRoots = rawRoots.map(root => { + try { + return fs.realpathSync(root); + } catch { + return root; + } + }); + + return this.allowedRoots; + } + + private isPathAllowed(candidate: string): boolean { + const allowed = this.getAllowedRoots(); + const isWindows = process.platform === 'win32'; + const normCandidate = isWindows ? candidate.toLowerCase() : candidate; + + return allowed.some(root => { + const normRoot = isWindows ? root.toLowerCase() : root; + const relative = path.relative(normRoot, normCandidate); + return !relative.startsWith('..') && !path.isAbsolute(relative); + }); + } + + /** + * Helper to try resolving a path locally without side-effects (errors) + */ + private async tryResolveLocal(inc: string, currentSource: string, context?: ParserContext, projectPath?: string): Promise<{ path: string, content: string } | undefined> { + const cleanInc = inc.startsWith('/') ? inc.substring(1) : inc; + const workspaceFolders = vscode.workspace.workspaceFolders; + + const candidates: string[] = []; + // Absolute path + if (path.isAbsolute(inc)) { + candidates.push(path.normalize(inc)); + } else { + // Check allowed/trusted roots first (this includes workspace folders, entryDirectory, and trustedIncludeRoot) + const allowedRoots = this.getAllowedRoots(); + for (const root of allowedRoots) { + candidates.push(path.normalize(path.join(root, cleanInc))); + if (projectPath) { + const projectName = path.basename(projectPath); + candidates.push(path.normalize(path.join(root, projectName, cleanInc))); + } + } + // Workspace relative (fallback) + if (workspaceFolders && workspaceFolders.length > 0) { + for (const folder of workspaceFolders) { + candidates.push(path.normalize(path.join(folder.uri.fsPath, cleanInc))); + if (projectPath) { + const projectName = path.basename(projectPath); + candidates.push(path.normalize(path.join(folder.uri.fsPath, projectName, cleanInc))); + } + } + } + // currentSource relative + if (path.isAbsolute(currentSource)) { + candidates.push(path.normalize(path.join(path.dirname(currentSource), cleanInc))); + if (projectPath) { + const projectName = path.basename(projectPath); + candidates.push(path.normalize(path.join(path.dirname(currentSource), '..', projectName, cleanInc))); + } + } + } + + for (const candidate of candidates) { + try { + // Resolve symlinks to prevent path traversal and local file inclusion + let realCandidate = candidate; + try { + realCandidate = await fs.promises.realpath(candidate); + } catch { + // File may not exist yet, access checks below will handle it + } + + if (!this.isPathAllowed(realCandidate)) { + continue; + } + + await fs.promises.access(realCandidate, fs.constants.R_OK); + const content = await fs.promises.readFile(realCandidate, 'utf8'); + return { path: realCandidate, content }; + } catch { + continue; + } + } + return undefined; + } + + private async resolveLocalInclude(inc: string, currentSource: string, depth: number, parentNode: IncludeNode, context?: ParserContext, componentOrigin?: ComponentOrigin) { + try { + // Try resolving locally first, allowing local overrides and workspace matching + const resolved = await this.tryResolveLocal(inc, currentSource, context); + if (resolved) { + if (!this.includedSources.includes(resolved.path)) { + this.includedSources.push(resolved.path); + } + const node = { name: resolved.path, children: [] }; + parentNode.children.push(node); + await this.parseRecursive(resolved.content, resolved.path, depth, node, context); + return; + } + + // If we're inside a fetched component/project template, resolve local: via GitLab API + if (componentOrigin) { + const cleanPath = inc.replace(/^\//, ''); + const targetName = `local:${inc}`; + if (!this.includedSources.includes(targetName)) { + this.includedSources.push(targetName); + } + const cacheManager = getComponentCacheManager(); + const content = await cacheManager.fetchAndCacheRawTemplate( + componentOrigin.gitlabInstance, + componentOrigin.projectPath, + cleanPath, + componentOrigin.ref + ); + if (content && typeof content === 'string' && !content.includes('{"message":"404 Project Not Found"}')) { + const node = { name: targetName, children: [] }; + parentNode.children.push(node); + await this.parseRecursive(content, targetName, depth, node, context, componentOrigin); + } else { + this.errors.push(`Could not fetch local file ${inc} from ${componentOrigin.projectPath}`); + } + return; + } + + this.errors.push(`Cannot find local file ${inc} (checked workspace root and relative to ${path.basename(currentSource)})`); + } catch (err) { + this.errors.push(`Failed to read local file ${inc}: ${err instanceof Error ? err.message : String(err)}`); + } + } + + private buildGraph(): PipelineGraph { + // Build final list of stages. If customStages is empty, use DEFAULT_STAGES + // Actually, GitLab merges custom stages with .pre and .post. + const finalStages: PipelineStage[] = []; + + let orderedStages = [...this.customStages]; + if (orderedStages.length === 0) { + orderedStages = [...DEFAULT_STAGES]; + } else { + if (!orderedStages.includes('.pre')) orderedStages.unshift('.pre'); + if (!orderedStages.includes('.post')) orderedStages.push('.post'); + } + + // Ensure all jobs have their stages created, even if they defined a stage that isn't in `stages`. + // Insert before .post so .post always remains last, matching GitLab CI behaviour. + const jobStages = new Set(this.allJobs.map(j => j.stage)); + const postIdx = orderedStages.indexOf('.post'); + for (const s of jobStages) { + if (!orderedStages.includes(s)) { + if (postIdx >= 0) { + orderedStages.splice(postIdx, 0, s); + } else { + orderedStages.push(s); + } + } + } + + for (const stageName of orderedStages) { + const jobsInStage = this.allJobs.filter(j => j.stage === stageName); + finalStages.push({ + name: stageName, + jobs: jobsInStage, + isImplicit: DEFAULT_STAGES.includes(stageName) && !this.customStages.includes(stageName) + }); + } + + return { + stages: finalStages, + includedSources: this.includedSources, + includeTree: this.includeTree, + errors: this.errors + }; + } +} diff --git a/src/providers/pipelineParserCommand.ts b/src/providers/pipelineParserCommand.ts new file mode 100644 index 00000000..21b32874 --- /dev/null +++ b/src/providers/pipelineParserCommand.ts @@ -0,0 +1,228 @@ +import * as vscode from 'vscode'; +import { PipelineParser, PipelineGraph, PipelineJob } from '../parsers/pipelineParser'; +import { getComponentService } from '../services/component/componentService'; + +export class PipelineParserCommand { + private context: vscode.ExtensionContext; + + constructor(context: vscode.ExtensionContext) { + this.context = context; + } + + public async parseAndShowTui(document?: vscode.TextDocument) { + if (!document) { + vscode.window.showErrorMessage("No active GitLab CI document to parse."); + return; + } + + const content = document.getText(); + const sourceName = document.uri.fsPath; + + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + const customVariables = config.get>('customVariables', {}); + const activePolicyOverride = config.get('parser.activePolicyOverride', ''); + const gitlabUrl = config.get('gitlabUrl', 'https://gitlab.com'); + let projectPath = config.get('projectPath', ''); + + // Try to auto-discover project path from local .git/config if not set in VS Code settings + if (!projectPath && sourceName && require('path').isAbsolute(sourceName)) { + try { + const { getProjectPathFromLocalFile } = require('../utils/gitUtils'); + const discoveredPath = await getProjectPathFromLocalFile(sourceName); + if (discoveredPath) { + projectPath = discoveredPath; + } + } catch (e) { + // Ignore + } + } + + let gitlabInstance = 'gitlab.com'; + try { + gitlabInstance = new URL(gitlabUrl).hostname; + } catch { + gitlabInstance = gitlabUrl.replace(/^https?:\/\//, '').split('/')[0]; + } + + const parserContext = { gitlabInstance, projectPath, customVariables, activePolicyOverride }; + let includesToProcess: string[] = []; + let pepWarning: string | undefined; + let pepInfo: string | undefined; + + vscode.window.withProgress({ + location: vscode.ProgressLocation.Notification, + title: "Parsing GitLab Pipeline...", + cancellable: false + }, async (progress) => { + try { + if (projectPath) { + const componentService = getComponentService(); + const token = await componentService.getTokenForProject(gitlabInstance); + + const pepOverride = config.get('parser.pepProjectPathOverride'); + let linkedProject: string | undefined = undefined; + + if (pepOverride) { + linkedProject = pepOverride; + } else { + linkedProject = await componentService.fetchLinkedSecurityPolicyProject(gitlabInstance, projectPath, token || ''); + } + + if (linkedProject) { + includesToProcess.push(`project:${linkedProject}:.gitlab/security-policies/policy.yml`); + pepInfo = pepOverride + ? `Loaded PEP from explicit override project '${linkedProject}'.` + : `Loaded PEP from linked project '${linkedProject}'.`; + } else { + pepWarning = `No linked Security Policy Project (PEP) was returned by GitLab for '${projectPath}'. Defaulting to local repository fallback.`; + includesToProcess.push(`project:${projectPath}:.gitlab/security-policies/policy.yml`); + } + } else { + pepWarning = `Cannot automatically discover Pipeline Execution Policies (PEP): 'gitlabComponentHelper.projectPath' is not configured.`; + } + + const extraIncludes = includesToProcess.map(entry => { + if (entry.startsWith('project:')) { + const parts = entry.slice('project:'.length).split(':'); + const project = parts[0]; + const fileAndRef = parts[1] || ''; + const atIdx = fileAndRef.lastIndexOf('@'); + const file = atIdx >= 0 ? fileAndRef.slice(0, atIdx) : fileAndRef; + const ref = atIdx >= 0 ? fileAndRef.slice(atIdx + 1) : 'HEAD'; + return { project, file, ref }; + } + return { local: entry }; + }); + + const parser = new PipelineParser(10); + const graph = await parser.parse(content, sourceName, parserContext, extraIncludes.length > 0 ? extraIncludes : undefined); + + if (pepWarning) graph.errors.push(pepWarning); + if (pepInfo) graph.errors.push(`ℹ️ Info: ${pepInfo}`); + + // Step 2: Show QuickPick for filtering sources + const allSources = new Set(); + const extractSources = (node: any) => { + const match = node.name.match(/\(Local Override: (.*?)\)$/); + const raw = match ? match[1] : node.name; + allSources.add(raw.replace(/\\\\/g, '/')); + if (node.children) { + for (const child of node.children) extractSources(child); + } + }; + if (graph.includeTree) extractSources(graph.includeTree); + + const quickPickItems: vscode.QuickPickItem[] = Array.from(allSources).map(src => ({ + label: src, + picked: true + })); + + // Ensure there is something to pick, else skip filtering + let hiddenSources = new Set(); + if (quickPickItems.length > 1) { + const selected = await vscode.window.showQuickPick(quickPickItems, { + canPickMany: true, + placeHolder: 'Select included files to visualize (uncheck to hide their jobs)' + }); + if (!selected) return; // User cancelled + + const selectedSources = new Set(selected.map(s => s.label)); + hiddenSources = new Set(Array.from(allSources).filter(s => !selectedSources.has(s))); + } + + // Step 3: Generate Markdown output + await this.generateMarkdownOutput(graph, sourceName, hiddenSources); + + } catch (error) { + vscode.window.showErrorMessage(`Failed to parse pipeline: ${error instanceof Error ? error.message : String(error)}`); + } + }); + } + + private async generateMarkdownOutput(graph: PipelineGraph, sourceName: string, hiddenSources: Set) { + const effectiveHidden = new Set(); + + if (graph.includeTree) { + const traverse = (node: any, parentHidden: boolean) => { + const match = node.name.match(/\(Local Override: (.*?)\)$/); + const raw = match ? match[1] : node.name; + const source = raw.replace(/\\\\/g, '/'); + + const isHidden = parentHidden || hiddenSources.has(source); + if (isHidden) effectiveHidden.add(source); + + if (node.children) { + for (const child of node.children) traverse(child, isHidden); + } + }; + traverse(graph.includeTree, false); + } + + let md = `# GitLab Pipeline Visualization\n\n`; + md += `**Source:** \`${sourceName}\`\n\n`; + + if (graph.errors.length > 0) { + md += `## Warnings / Info\n`; + graph.errors.forEach(err => { + md += `- ${err}\n`; + }); + md += `\n`; + } + + md += `## Pipeline Stages & Jobs\n\n`; + + const visibleStages = graph.stages.filter(s => !s.isImplicit || s.jobs.length > 0); + + visibleStages.forEach(stage => { + const visibleJobs = stage.jobs.filter(job => !effectiveHidden.has(job.source)); + + if (visibleJobs.length === 0 && stage.isImplicit) return; + + md += `### Stage: ${stage.name} ${stage.isImplicit ? '(Implicit)' : ''}\n`; + + if (visibleJobs.length === 0) { + md += `*No jobs in this stage*\n`; + } else { + visibleJobs.forEach(job => { + md += `- **${job.name}** *(Source: \`${job.source}\`)*\n`; + }); + } + md += `\n`; + }); + + md += `## Included Sources Tree\n\n`; + md += "```text\n"; + + const renderIncludeTree = (node: any, depth: number = 0, isLast: boolean = true, prefix: string = '') => { + if (depth > 15) return `${prefix}${isLast ? '└── ' : '├── '}... (max depth reached)\n`; + + const connector = depth === 0 ? '' : (isLast ? '└── ' : '├── '); + let line = `${prefix}${connector}${node.name}\n`; + + const newPrefix = depth === 0 ? '' : prefix + (isLast ? ' ' : '│ '); + + if (node.children && node.children.length > 0) { + node.children.forEach((child: any, index: number) => { + line += renderIncludeTree(child, depth + 1, index === node.children.length - 1, newPrefix); + }); + } + return line; + }; + + if (graph.includeTree) { + md += renderIncludeTree(graph.includeTree); + } else { + md += `No includes found.\n`; + } + md += "```\n"; + + // Provide an invisible data block for the piggybacking component + md += `\n\n`; + + const document = await vscode.workspace.openTextDocument({ + content: md, + language: 'markdown' + }); + await vscode.window.showTextDocument(document, { preview: false }); + } +} diff --git a/src/providers/pipelineParserCommandRegistrations.ts b/src/providers/pipelineParserCommandRegistrations.ts new file mode 100644 index 00000000..9569c05d --- /dev/null +++ b/src/providers/pipelineParserCommandRegistrations.ts @@ -0,0 +1,127 @@ +import * as vscode from 'vscode'; +import { PipelineParserCommand } from './pipelineParserCommand'; +import { getComponentService } from '../services/component'; + +async function safeUpdateConfig(config: vscode.WorkspaceConfiguration, section: string, value: any, successMessage?: string) { + try { + await config.update(section, value, vscode.ConfigurationTarget.Workspace); + if (successMessage) vscode.window.showInformationMessage(successMessage); + } catch (error) { + vscode.window.showErrorMessage(`Failed to update settings (you might be in a restricted workspace): ${error}`); + } +} + +export function registerPipelineParserCommands(context: vscode.ExtensionContext) { + const parserCommand = new PipelineParserCommand(context); + + context.subscriptions.push( + vscode.commands.registerCommand('gitlab-component-helper.parsePipeline', async () => { + await parserCommand.parseAndShowTui(vscode.window.activeTextEditor?.document); + }) + ); + + context.subscriptions.push( + vscode.commands.registerCommand('gitlab-component-helper.selectPolicyOverride', async () => { + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + const projectPath = config.get('projectPath', ''); + + if (!projectPath) { + vscode.window.showErrorMessage("Cannot list policies: 'gitlabComponentHelper.projectPath' is not configured."); + return; + } + + const gitlabUrl = config.get('gitlabUrl', 'https://gitlab.com'); + const gitlabInstance = gitlabUrl.replace(/^https?:\/\//, '').split('/')[0]; + const componentService = getComponentService(); + const token = await componentService.getTokenForProject(gitlabInstance); + + vscode.window.withProgress({ + location: vscode.ProgressLocation.Notification, + title: "Fetching available Pipeline Execution Policies (PEP)..." + }, async () => { + try { + const policies = await componentService.fetchPipelineExecutionPolicies(gitlabInstance, projectPath, token || ''); + + if (policies.length === 0) { + vscode.window.showInformationMessage(`No active Pipeline Execution Policies found for project '${projectPath}'.`); + return; + } + + const selected = await vscode.window.showQuickPick(policies, { + placeHolder: 'Select a Pipeline Execution Policy to override as active' + }); + + if (selected) { + await safeUpdateConfig(config, 'parser.activePolicyOverride', selected, `Active Policy override set to: ${selected}`); + } + } catch (error) { + vscode.window.showErrorMessage(`Failed to fetch policies: ${error}`); + } + }); + }) + ); + + context.subscriptions.push( + vscode.commands.registerCommand('gitlab-component-helper.setProjectPath', async () => { + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + const currentPath = config.get('projectPath', ''); + + const input = await vscode.window.showInputBox({ + prompt: 'Enter the GitLab project path (e.g., my-group/my-project)', + value: currentPath + }); + + if (input !== undefined) { + await safeUpdateConfig(config, 'projectPath', input, `Project path set to: ${input}`); + } + }) + ); + + context.subscriptions.push( + vscode.commands.registerCommand('gitlab-component-helper.selectLocalPepOverride', async () => { + const uris = await vscode.window.showOpenDialog({ + canSelectMany: false, + openLabel: 'Select Local Policy', + filters: { 'YAML': ['yml', 'yaml'] } + }); + + if (uris && uris.length > 0) { + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + let trustedRoot = config.get('trustedIncludeRoot', []); + + // Remove existing absolute paths (which represent local file overrides) + trustedRoot = trustedRoot.filter(inc => !require('path').isAbsolute(inc)); + + trustedRoot.push(uris[0].fsPath); + await safeUpdateConfig(config, 'trustedIncludeRoot', trustedRoot, `Local PEP override set to: ${uris[0].fsPath}`); + } + }) + ); + + context.subscriptions.push( + vscode.commands.registerCommand('gitlab-component-helper.clearPepOverrides', async () => { + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + let updated = false; + + if (config.get('parser.activePolicyOverride', '') !== '') { + await safeUpdateConfig(config, 'parser.activePolicyOverride', ''); + updated = true; + } + + let trustedRoot = config.get('trustedIncludeRoot', []); + const originalLength = trustedRoot.length; + trustedRoot = trustedRoot.filter(inc => !require('path').isAbsolute(inc)); + + if (trustedRoot.length !== originalLength) { + await safeUpdateConfig(config, 'trustedIncludeRoot', trustedRoot); + updated = true; + } + + if (updated) { + vscode.window.showInformationMessage('All PEP overrides cleared.'); + } else { + vscode.window.showInformationMessage('No active PEP overrides to clear.'); + } + }) + ); +} diff --git a/src/services/cache/componentCacheManager.ts b/src/services/cache/componentCacheManager.ts index ad92408d..32ea8285 100644 --- a/src/services/cache/componentCacheManager.ts +++ b/src/services/cache/componentCacheManager.ts @@ -35,6 +35,7 @@ export class ComponentCacheManager { private refreshInProgress = false; private sourceErrors: Map = new Map(); private context: vscode.ExtensionContext | null = null; + private rawTemplateCache: Map = new Map(); // Specialized cache modules private projectCache: ProjectCache; @@ -752,6 +753,36 @@ export class ComponentCacheManager { }; } + public getCachedComponents(): CachedComponent[] { + return this.components; + } + + /** + * Fetch and cache raw YAML templates to optimize include parsing + */ + public async fetchAndCacheRawTemplate( + gitlabInstance: string, + projectPath: string, + filePath: string, + version: string + ): Promise { + const cacheKey = `${gitlabInstance}:${projectPath}:${filePath}@${version}`; + const config = vscode.workspace.getConfiguration('gitlabComponentHelper'); + const cacheTime = config.get('cacheTime', 3600) * 1000; + + const cached = this.rawTemplateCache.get(cacheKey); + if (cached && Date.now() - cached.timestamp < cacheTime) { + this.logger.debug(`[ComponentCache] Returning cached raw template for ${cacheKey}`, 'ComponentCache'); + return cached.content; + } + + const componentService = getComponentService(); + const content = await componentService.fetchRawFile(gitlabInstance, projectPath, filePath, version); + + this.rawTemplateCache.set(cacheKey, { content, timestamp: Date.now() }); + return content; + } + /** * Get local fallback components when no sources are configured */ diff --git a/src/services/component/componentService.ts b/src/services/component/componentService.ts index e9d7a259..d769133b 100644 --- a/src/services/component/componentService.ts +++ b/src/services/component/componentService.ts @@ -184,11 +184,150 @@ export class ComponentService implements ComponentSource { ); } + public async fetchLinkedSecurityPolicyProject( + gitlabInstance: string, + projectPath: string, + token: string + ): Promise { + const cleanGitlabInstance = this.urlParser.cleanGitLabInstance(gitlabInstance); + const apiBaseUrl = `https://${cleanGitlabInstance}/api/graphql`; + + const headers: Record = {}; + if (token) { + headers['PRIVATE-TOKEN'] = token; + } + + try { + // First try project level + const projectQuery = ` + query getPolicyProject($fullPath: ID!) { + project(fullPath: $fullPath) { + securityPolicyProject { + fullPath + } + } + } + `; + + let response = await this.httpClient.fetchGraphQL( + apiBaseUrl, + projectQuery, + { fullPath: projectPath }, + { headers } + ); + + let policyProject = response?.data?.project?.securityPolicyProject?.fullPath; + if (policyProject) { + this.logger.debug(`[ComponentService] Found direct security policy project: ${policyProject}`); + return policyProject; + } + + // If not found, walk up the namespace (groups) to find inherited policies + const pathSegments = projectPath.split('/'); + pathSegments.pop(); // Remove the project name to get the parent group + + const groupQuery = ` + query getPolicyGroup($fullPath: ID!) { + group(fullPath: $fullPath) { + securityPolicyProject { + fullPath + } + } + } + `; + + while (pathSegments.length > 0) { + const groupPath = pathSegments.join('/'); + this.logger.debug(`[ComponentService] Checking parent group for policy project: ${groupPath}`); + + response = await this.httpClient.fetchGraphQL( + apiBaseUrl, + groupQuery, + { fullPath: groupPath }, + { headers } + ); + + policyProject = response?.data?.group?.securityPolicyProject?.fullPath; + if (policyProject) { + this.logger.debug(`[ComponentService] Found inherited security policy project from ${groupPath}: ${policyProject}`); + return policyProject; + } + + pathSegments.pop(); + } + + this.logger.debug(`[ComponentService] No security policy project found for ${projectPath} or its parent groups`); + return undefined; + } catch (error) { + this.logger.error(`Error fetching linked security policy project: ${error}`); + return undefined; + } + } + + public async fetchPipelineExecutionPolicies( + gitlabInstance: string, + projectPath: string, + token: string + ): Promise { + const cleanGitlabInstance = this.urlParser.cleanGitLabInstance(gitlabInstance); + const apiBaseUrl = `https://${cleanGitlabInstance}/api/graphql`; + + const query = ` + query getPolicies($fullPath: ID!) { + project(fullPath: $fullPath) { + pipelineExecutionPolicies { + nodes { + name + } + } + } + } + `; + + const headers: Record = {}; + if (token) { + headers['PRIVATE-TOKEN'] = token; + } + + try { + const response = await this.httpClient.fetchGraphQL( + apiBaseUrl, + query, + { fullPath: projectPath }, + { headers } + ); + + const nodes = response?.data?.project?.pipelineExecutionPolicies?.nodes || []; + return nodes.map((node: any) => node.name).filter(Boolean); + } catch (error) { + this.logger.error(`Error fetching pipeline execution policies: ${error}`); + return []; + } + } + // HTTP client delegation public async fetchJson(url: string, options?: { headers?: Record }): Promise { return this.httpClient.fetchJson(url, options); } + public async fetchRawFile( + gitlabInstance: string, + projectPath: string, + filePath: string, + ref: string = 'main' + ): Promise { + const cleanGitlabInstance = this.urlParser.cleanGitLabInstance(gitlabInstance); + const apiBaseUrl = `https://${cleanGitlabInstance}/api/v4`; + const url = `${apiBaseUrl}/projects/${encodeURIComponent( + projectPath + )}/repository/files/${encodeURIComponent(filePath)}/raw?ref=${ref}`; + + const token = await this.getTokenForProject(cleanGitlabInstance); + const headers = token ? { 'PRIVATE-TOKEN': token } : undefined; + + return this.httpClient.fetchText(url, { headers }); + } + private async fetchText(url: string): Promise { return this.httpClient.fetchText(url); } diff --git a/src/utils/gitlabVariables.ts b/src/utils/gitlabVariables.ts index 766e9a35..ba894bc4 100644 --- a/src/utils/gitlabVariables.ts +++ b/src/utils/gitlabVariables.ts @@ -1,266 +1,362 @@ -/** - * GitLab CI/CD predefined variables and utilities for handling them - */ - -export interface GitLabVariable { - name: string; - description: string; - example: string; - availableIn?: string[]; -} - -/** - * Common GitLab CI/CD predefined variables - * Reference: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html - */ -export const GITLAB_PREDEFINED_VARIABLES: GitLabVariable[] = [ - { - name: 'CI_API_V4_URL', - description: 'The GitLab API v4 root URL', - example: 'https://gitlab.example.com/api/v4' - }, - { - name: 'CI_BUILDS_DIR', - description: 'The top-level directory where builds are executed', - example: '/builds' - }, - { - name: 'CI_COMMIT_BRANCH', - description: 'The commit branch name. Available in branch pipelines', - example: 'main' - }, - { - name: 'CI_COMMIT_REF_NAME', - description: 'The branch or tag name for which project is built', - example: 'main' - }, - { - name: 'CI_COMMIT_REF_SLUG', - description: 'CI_COMMIT_REF_NAME in lowercase, shortened to 63 bytes, and with everything except 0-9 and a-z replaced with -', - example: 'main' - }, - { - name: 'CI_COMMIT_SHA', - description: 'The commit revision the project is built for', - example: '1ecfd275763eff1d6b4844ea3168962458c9f27a' - }, - { - name: 'CI_COMMIT_SHORT_SHA', - description: 'The first eight characters of CI_COMMIT_SHA', - example: '1ecfd275' - }, - { - name: 'CI_COMMIT_TAG', - description: 'The commit tag name. Available only in pipelines for tags', - example: 'v1.0.0' - }, - { - name: 'CI_COMMIT_TITLE', - description: 'The title of the commit. The full first line of the message', - example: 'Add new feature' - }, - { - name: 'CI_PROJECT_ID', - description: 'The ID of the project', - example: '42' - }, - { - name: 'CI_PROJECT_NAME', - description: 'The name of the project', - example: 'my-project' - }, - { - name: 'CI_PROJECT_NAMESPACE', - description: 'The project namespace (username or group name)', - example: 'my-group' - }, - { - name: 'CI_PROJECT_PATH', - description: 'The project path with namespace', - example: 'my-group/my-project' - }, - { - name: 'CI_PROJECT_PATH_SLUG', - description: 'CI_PROJECT_PATH in lowercase, shortened to 63 bytes, and with everything except 0-9 and a-z replaced with -', - example: 'my-group-my-project' - }, - { - name: 'CI_PROJECT_ROOT_NAMESPACE', - description: 'The root project namespace (username or group name)', - example: 'my-group' - }, - { - name: 'CI_PROJECT_URL', - description: 'The HTTP(S) address to access project', - example: 'https://gitlab.example.com/my-group/my-project' - }, - { - name: 'CI_REGISTRY', - description: 'The address of the GitLab Container Registry', - example: 'registry.gitlab.example.com' - }, - { - name: 'CI_REGISTRY_IMAGE', - description: 'The address of the project\'s Container Registry', - example: 'registry.gitlab.example.com/my-group/my-project' - }, - { - name: 'CI_SERVER_FQDN', - description: 'The FQDN of the GitLab instance', - example: 'gitlab.example.com' - }, - { - name: 'CI_SERVER_HOST', - description: 'The host of the GitLab instance URL, without protocol and port', - example: 'gitlab.example.com' - }, - { - name: 'CI_SERVER_NAME', - description: 'The name of CI/CD server that coordinates jobs', - example: 'GitLab' - }, - { - name: 'CI_SERVER_PORT', - description: 'The port of the GitLab instance URL, without host and protocol', - example: '443' - }, - { - name: 'CI_SERVER_PROTOCOL', - description: 'The protocol of the GitLab instance URL, without host and port', - example: 'https' - }, - { - name: 'CI_SERVER_REVISION', - description: 'GitLab revision that schedules jobs', - example: '70606bf' - }, - { - name: 'CI_SERVER_URL', - description: 'The base URL of the GitLab instance, including protocol and port', - example: 'https://gitlab.example.com:8080' - }, - { - name: 'CI_SERVER_VERSION', - description: 'GitLab version that schedules jobs', - example: '13.12.0' - }, - { - name: 'CI_SERVER_VERSION_MAJOR', - description: 'GitLab major version that schedules jobs', - example: '13' - }, - { - name: 'CI_SERVER_VERSION_MINOR', - description: 'GitLab minor version that schedules jobs', - example: '12' - }, - { - name: 'CI_SERVER_VERSION_PATCH', - description: 'GitLab patch version that schedules jobs', - example: '0' - } -]; - -/** - * Detects GitLab predefined variables in a string - */ -export function detectGitLabVariables(text: string): string[] { - const variablePattern = /\$([A-Z_][A-Z0-9_]*)/g; - const matches = text.match(variablePattern); - - if (!matches) { - return []; - } - - const variables = matches.map(match => match.substring(1)); // Remove the $ - const predefinedVariableNames = GITLAB_PREDEFINED_VARIABLES.map(v => v.name); - - return variables.filter(variable => predefinedVariableNames.includes(variable)); -} - -/** - * Checks if a string contains GitLab predefined variables - */ -export function containsGitLabVariables(text: string): boolean { - return detectGitLabVariables(text).length > 0; -} - -/** - * Expands GitLab variables specifically in component URLs, ensuring proper URL formatting - */ -export function expandComponentUrl(componentUrl: string, context?: { - gitlabInstance?: string; - projectPath?: string; - serverUrl?: string; - commitSha?: string; // Optionally provide a commit SHA for expansion -}): string { - let expanded = componentUrl; - - if (context) { - // Handle URL expansion carefully to maintain proper URL structure - if (context.gitlabInstance) { - // For component URLs that start with $CI_SERVER_FQDN, we need to ensure https:// is added - if (expanded.startsWith('$CI_SERVER_FQDN/')) { - expanded = expanded.replace(/^\$CI_SERVER_FQDN\//, `https://${context.gitlabInstance}/`); - } else { - // For other cases, do normal replacement - expanded = expanded.replace(/\$CI_SERVER_FQDN/g, context.gitlabInstance); - expanded = expanded.replace(/\$CI_SERVER_HOST/g, context.gitlabInstance); - expanded = expanded.replace(/\$CI_SERVER_URL/g, context.serverUrl || `https://${context.gitlabInstance}`); - } - } - - if (context.projectPath) { - expanded = expanded.replace(/\$CI_PROJECT_PATH/g, context.projectPath); - - // Extract namespace and project name - const parts = context.projectPath.split('/'); - if (parts.length >= 2) { - const namespace = parts.slice(0, -1).join('/'); - const projectName = parts[parts.length - 1]; - - expanded = expanded.replace(/\$CI_PROJECT_NAMESPACE/g, namespace); - expanded = expanded.replace(/\$CI_PROJECT_NAME/g, projectName); - expanded = expanded.replace(/\$CI_PROJECT_ROOT_NAMESPACE/g, parts[0]); - } - } - - // Support $CI_COMMIT_SHA expansion - if (expanded.includes('$CI_COMMIT_SHA')) { - // Use context.commitSha if provided, otherwise fallback to a placeholder or branch name - const shaValue = context.commitSha || '[current-branch-or-sha]'; - expanded = expanded.replace(/\$CI_COMMIT_SHA/g, shaValue); - } - } - - // Ensure the URL starts with https:// if it doesn't already have a protocol - if (!expanded.match(/^https?:\/\//)) { - // If it looks like a domain/path pattern, add https:// - if (expanded.match(/^[a-zA-Z0-9.-]+\//)) { - expanded = `https://${expanded}`; - } - } - - return expanded; -} - -/** - * Gets variable information for completion/documentation - */ -export function getVariableInfo(variableName: string): GitLabVariable | undefined { - return GITLAB_PREDEFINED_VARIABLES.find(v => v.name === variableName); -} - -/** - * Provides completion suggestions for GitLab variables - */ -export function getVariableCompletions(prefix: string = ''): GitLabVariable[] { - if (!prefix) { - return GITLAB_PREDEFINED_VARIABLES; - } - - const upperPrefix = prefix.toUpperCase(); - return GITLAB_PREDEFINED_VARIABLES.filter(v => - v.name.includes(upperPrefix) || v.description.toLowerCase().includes(prefix.toLowerCase()) - ); -} +/** + * GitLab CI/CD predefined variables and utilities for handling them + */ + +export interface GitLabVariable { + name: string; + description: string; + example: string; + availableIn?: string[]; +} + +/** + * Common GitLab CI/CD predefined variables + * Reference: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html + */ +export const GITLAB_PREDEFINED_VARIABLES: GitLabVariable[] = [ + { + name: 'CI_API_V4_URL', + description: 'The GitLab API v4 root URL', + example: 'https://gitlab.example.com/api/v4' + }, + { + name: 'CI_BUILDS_DIR', + description: 'The top-level directory where builds are executed', + example: '/builds' + }, + { + name: 'CI_COMMIT_BRANCH', + description: 'The commit branch name. Available in branch pipelines', + example: 'main' + }, + { + name: 'CI_COMMIT_REF_NAME', + description: 'The branch or tag name for which project is built', + example: 'main' + }, + { + name: 'CI_COMMIT_REF_SLUG', + description: 'CI_COMMIT_REF_NAME in lowercase, shortened to 63 bytes, and with everything except 0-9 and a-z replaced with -', + example: 'main' + }, + { + name: 'CI_COMMIT_SHA', + description: 'The commit revision the project is built for', + example: '1ecfd275763eff1d6b4844ea3168962458c9f27a' + }, + { + name: 'CI_COMMIT_SHORT_SHA', + description: 'The first eight characters of CI_COMMIT_SHA', + example: '1ecfd275' + }, + { + name: 'CI_COMMIT_TAG', + description: 'The commit tag name. Available only in pipelines for tags', + example: 'v1.0.0' + }, + { + name: 'CI_COMMIT_TITLE', + description: 'The title of the commit. The full first line of the message', + example: 'Add new feature' + }, + { + name: 'CI_PROJECT_ID', + description: 'The ID of the project', + example: '42' + }, + { + name: 'CI_PROJECT_NAME', + description: 'The name of the project', + example: 'my-project' + }, + { + name: 'CI_PROJECT_NAMESPACE', + description: 'The project namespace (username or group name)', + example: 'my-group' + }, + { + name: 'CI_PROJECT_PATH', + description: 'The project path with namespace', + example: 'my-group/my-project' + }, + { + name: 'CI_PROJECT_PATH_SLUG', + description: 'CI_PROJECT_PATH in lowercase, shortened to 63 bytes, and with everything except 0-9 and a-z replaced with -', + example: 'my-group-my-project' + }, + { + name: 'CI_PROJECT_ROOT_NAMESPACE', + description: 'The root project namespace (username or group name)', + example: 'my-group' + }, + { + name: 'CI_PROJECT_URL', + description: 'The HTTP(S) address to access project', + example: 'https://gitlab.example.com/my-group/my-project' + }, + { + name: 'CI_REGISTRY', + description: 'The address of the GitLab Container Registry', + example: 'registry.gitlab.example.com' + }, + { + name: 'CI_REGISTRY_IMAGE', + description: 'The address of the project\'s Container Registry', + example: 'registry.gitlab.example.com/my-group/my-project' + }, + { + name: 'CI_SERVER_FQDN', + description: 'The FQDN of the GitLab instance', + example: 'gitlab.example.com' + }, + { + name: 'CI_SERVER_HOST', + description: 'The host of the GitLab instance URL, without protocol and port', + example: 'gitlab.example.com' + }, + { + name: 'CI_SERVER_NAME', + description: 'The name of CI/CD server that coordinates jobs', + example: 'GitLab' + }, + { + name: 'CI_SERVER_PORT', + description: 'The port of the GitLab instance URL, without host and protocol', + example: '443' + }, + { + name: 'CI_SERVER_PROTOCOL', + description: 'The protocol of the GitLab instance URL, without host and port', + example: 'https' + }, + { + name: 'CI_SERVER_REVISION', + description: 'GitLab revision that schedules jobs', + example: '70606bf' + }, + { + name: 'CI_SERVER_URL', + description: 'The base URL of the GitLab instance, including protocol and port', + example: 'https://gitlab.example.com:8080' + }, + { + name: 'CI_SERVER_VERSION', + description: 'GitLab version that schedules jobs', + example: '13.12.0' + }, + { + name: 'CI_SERVER_VERSION_MAJOR', + description: 'GitLab major version that schedules jobs', + example: '13' + }, + { + name: 'CI_SERVER_VERSION_MINOR', + description: 'GitLab minor version that schedules jobs', + example: '12' + }, + { + name: 'CI_SERVER_VERSION_PATCH', + description: 'GitLab patch version that schedules jobs', + example: '0' + } +]; + +/** + * Detects GitLab predefined variables in a string + */ +export function detectGitLabVariables(text: string): string[] { + const variablePattern = /\$([A-Z_][A-Z0-9_]*)/g; + const matches = text.match(variablePattern); + + if (!matches) { + return []; + } + + const variables = matches.map(match => match.substring(1)); // Remove the $ + const predefinedVariableNames = GITLAB_PREDEFINED_VARIABLES.map(v => v.name); + + return variables.filter(variable => predefinedVariableNames.includes(variable)); +} + +/** + * Checks if a string contains GitLab predefined variables + */ +export function containsGitLabVariables(text: string): boolean { + return detectGitLabVariables(text).length > 0; +} + +/** + * Expands GitLab variables in a component URL using context from the current workspace/project + * This is a best-effort expansion for development purposes + */ +export function expandGitLabVariables(text: string, context?: { + gitlabInstance?: string; + projectPath?: string; + serverUrl?: string; + customVariables?: Record; +}): string { + let expanded = text; + + if (context) { + if (context.customVariables) { + for (const [key, value] of Object.entries(context.customVariables)) { + const escapedKey = escapeRegExp(key); + expanded = expanded.replace(new RegExp(`\\$${escapedKey}\\b`, 'g'), String(value)); + expanded = expanded.replace(new RegExp(`\\$\\{${escapedKey}\\}`, 'g'), String(value)); + } + } + // Expand common variables based on context + if (context.gitlabInstance) { + expanded = expanded.replace(/\$CI_SERVER_FQDN/g, context.gitlabInstance); + expanded = expanded.replace(/\$CI_SERVER_HOST/g, context.gitlabInstance); + expanded = expanded.replace(/\$CI_SERVER_URL/g, context.serverUrl || `https://${context.gitlabInstance}`); + } + + if (context.projectPath) { + expanded = expanded.replace(/\$CI_PROJECT_PATH/g, context.projectPath); + + // Extract namespace and project name + const parts = context.projectPath.split('/'); + if (parts.length >= 2) { + const namespace = parts.slice(0, -1).join('/'); + const projectName = parts[parts.length - 1]; + + expanded = expanded.replace(/\$CI_PROJECT_NAMESPACE/g, namespace); + expanded = expanded.replace(/\$CI_PROJECT_NAME/g, projectName); + expanded = expanded.replace(/\$CI_PROJECT_ROOT_NAMESPACE/g, parts[0]); + } + } + } + + return expanded; +} + +/** + * Expands GitLab variables specifically in component URLs, ensuring proper URL formatting + */ +export function expandComponentUrl(componentUrl: string, context?: { + gitlabInstance?: string; + projectPath?: string; + serverUrl?: string; + commitSha?: string; // Optionally provide a commit SHA for expansion + customVariables?: Record; +}): string { + let expanded = componentUrl; + + if (context) { + if (context.customVariables) { + for (const [key, value] of Object.entries(context.customVariables)) { + const escapedKey = escapeRegExp(key); + expanded = expanded.replace(new RegExp(`\\$${escapedKey}\\b`, 'g'), String(value)); + expanded = expanded.replace(new RegExp(`\\$\\{${escapedKey}\\}`, 'g'), String(value)); + } + } + // Handle URL expansion carefully to maintain proper URL structure + if (context.gitlabInstance) { + // For component URLs that start with $CI_SERVER_FQDN, we need to ensure https:// is added + if (expanded.startsWith('$CI_SERVER_FQDN/')) { + expanded = expanded.replace(/^\$CI_SERVER_FQDN\//, `https://${context.gitlabInstance}/`); + } else { + // For other cases, do normal replacement + expanded = expanded.replace(/\$CI_SERVER_FQDN/g, context.gitlabInstance); + expanded = expanded.replace(/\$CI_SERVER_HOST/g, context.gitlabInstance); + expanded = expanded.replace(/\$CI_SERVER_URL/g, context.serverUrl || `https://${context.gitlabInstance}`); + } + } + + if (context.projectPath) { + expanded = expanded.replace(/\$CI_PROJECT_PATH/g, context.projectPath); + + // Extract namespace and project name + const parts = context.projectPath.split('/'); + if (parts.length >= 2) { + const namespace = parts.slice(0, -1).join('/'); + const projectName = parts[parts.length - 1]; + + expanded = expanded.replace(/\$CI_PROJECT_NAMESPACE/g, namespace); + expanded = expanded.replace(/\$CI_PROJECT_NAME/g, projectName); + expanded = expanded.replace(/\$CI_PROJECT_ROOT_NAMESPACE/g, parts[0]); + } + } + + // Support $CI_COMMIT_SHA expansion + if (expanded.includes('$CI_COMMIT_SHA')) { + // Use context.commitSha if provided, otherwise fallback to a placeholder or branch name + const shaValue = context.commitSha || '[current-branch-or-sha]'; + expanded = expanded.replace(/\$CI_COMMIT_SHA/g, shaValue); + } + } + + // Ensure the URL starts with https:// if it doesn't already have a protocol + if (!expanded.match(/^https?:\/\//)) { + // If it looks like a domain/path pattern, add https:// + if (expanded.match(/^[a-zA-Z0-9.-]+\//)) { + expanded = `https://${expanded}`; + } + } + + return expanded; +} + +/** + * Validates that a component URL with variables can be properly resolved + */ +export function validateComponentUrlWithVariables(url: string): { + isValid: boolean; + unresolvedVariables: string[]; + suggestions: string[]; +} { + const variables = detectGitLabVariables(url); + const unresolvedVariables: string[] = []; + const suggestions: string[] = []; + + for (const variable of variables) { + const varInfo = GITLAB_PREDEFINED_VARIABLES.find(v => v.name === variable); + if (varInfo) { + // Check if this is a variable that can be reasonably resolved in development + if (['CI_SERVER_FQDN', 'CI_SERVER_HOST', 'CI_SERVER_URL', 'CI_PROJECT_PATH', 'CI_PROJECT_NAMESPACE', 'CI_PROJECT_NAME'].includes(variable)) { + suggestions.push(`Consider setting ${variable} context or using a literal value for development`); + } else { + unresolvedVariables.push(variable); + suggestions.push(`${variable}: ${varInfo.description} (example: ${varInfo.example})`); + } + } else { + unresolvedVariables.push(variable); + suggestions.push(`Unknown variable: ${variable}`); + } + } + + return { + isValid: unresolvedVariables.length === 0, + unresolvedVariables, + suggestions + }; +} + +/** + * Gets variable information for completion/documentation + */ +export function getVariableInfo(variableName: string): GitLabVariable | undefined { + return GITLAB_PREDEFINED_VARIABLES.find(v => v.name === variableName); +} + +/** + * Provides completion suggestions for GitLab variables + */ +export function getVariableCompletions(prefix: string = ''): GitLabVariable[] { + if (!prefix) { + return GITLAB_PREDEFINED_VARIABLES; + } + + const upperPrefix = prefix.toUpperCase(); + return GITLAB_PREDEFINED_VARIABLES.filter(v => + v.name.includes(upperPrefix) || v.description.toLowerCase().includes(prefix.toLowerCase()) + ); +} + +/** + * Escapes special characters in a string for use in a regular expression + */ +function escapeRegExp(string: string): string { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/src/utils/httpClient.ts b/src/utils/httpClient.ts index 128e3098..d022bb20 100644 --- a/src/utils/httpClient.ts +++ b/src/utils/httpClient.ts @@ -12,6 +12,8 @@ interface RequestOptions { retryAttempts?: number; headers?: Record; retryDelay?: number; + method?: string; + body?: string; } /** @@ -234,7 +236,7 @@ export class HttpClient { */ private async makeRequest( url: string, - options: { timeout: number; headers: Record } + options: { timeout: number; headers: Record; method?: string; body?: string } ): Promise { const { body } = await this.makeRequestWithHeaders(url, options); return body; @@ -252,7 +254,7 @@ export class HttpClient { */ private makeRequestWithHeaders( url: string, - options: { timeout: number; headers: Record } + options: { timeout: number; headers: Record; method?: string; body?: string } ): Promise<{ body: string; headers: Record }> { return new Promise((resolve, reject) => { try { @@ -264,7 +266,7 @@ export class HttpClient { hostname: urlObj.hostname, port: urlObj.port || (isHttps ? 443 : 80), path: urlObj.pathname + urlObj.search, - method: 'GET', + method: options.method || 'GET', headers: options.headers, timeout: options.timeout }; @@ -304,6 +306,9 @@ export class HttpClient { reject(new NetworkError(error.message, { cause: error })); }); + if (options.body) { + req.write(options.body); + } req.end(); } catch (error) { reject(new NetworkError(extractMessage(error), { cause: toError(error) })); @@ -423,6 +428,15 @@ export class HttpClient { return results; } + public async fetchGraphQL(url: string, query: string, variables: any, options: RequestOptions = {}): Promise { + const body = JSON.stringify({ query, variables }); + const headers = { + ...options.headers, + 'Content-Type': 'application/json' + }; + return this.fetchJson(url, { ...options, method: 'POST', body, headers }); + } + // Get request deduplication statistics getDeduplicationStats(): { pendingCount: number; pendingKeys: string[] } { return this.deduplicator.getStats(); diff --git a/tests/unit/pipelineParser.test.js b/tests/unit/pipelineParser.test.js new file mode 100644 index 00000000..cc653c92 --- /dev/null +++ b/tests/unit/pipelineParser.test.js @@ -0,0 +1,361 @@ +/** + * Pipeline Parser Unit Tests + * + * Tests stage merging, circular includes, and URL interpolation. + */ + +const assert = require('assert'); +const esbuild = require('esbuild'); +const fs = require('fs'); +const path = require('path'); +const Module = require('module'); + +console.log('=== Pipeline Parser Tests ==='); + +// Mock VS Code API and ComponentService before loading the parser. +// A single hook handles all mocked modules; the second definition was +// overwriting the first and losing onDidChangeConfiguration. +const originalRequire = Module.prototype.require; + +const mockComponentService = { + httpClient: { + fetchText: async (url) => { + if (url.includes('remote-a.yml')) { + return `include:\n - remote: https://example.com/remote-b.yml\njobA:\n script: echo "A"`; + } + if (url.includes('remote-b.yml')) { + return `include:\n - remote: https://example.com/remote-a.yml\njobB:\n script: echo "B"`; + } + if (url.includes('my-remote.yml')) { + return `jobRemote:\n script: echo "remote"`; + } + return ''; + } + } +}; + +const vscodeMock = { + workspace: { + getConfiguration: (section) => ({ + get: (key, defaultValue) => { + if (key === 'trustedIncludeRoot') { + // For Test 5/6, we trust the fixtures directory + return [path.join(__dirname, '../../tests/fixtures')]; + } + if (key === 'logLevel') return 'info'; + return defaultValue; + } + }), + workspaceFolders: [], + onDidChangeConfiguration: () => ({ dispose: () => { } }) + }, + Uri: { parse: (s) => ({ fsPath: s }), file: (s) => ({ fsPath: s }), joinPath: () => ({}) }, + window: { createOutputChannel: () => ({ appendLine: () => { } }) }, + commands: {} +}; + +Module.prototype.require = function (id) { + if (id === 'vscode') { return vscodeMock; } + if (id.includes('componentService')) { return { getComponentService: () => mockComponentService }; } + if (id.includes('componentCacheManager')) { return { getComponentCacheManager: () => ({ getComponents: async () => [], fetchAndCacheRawTemplate: async () => null }) }; } + return originalRequire.apply(this, arguments); +}; + + +async function runTests() { + let passed = 0; + let failed = 0; + + // Dynamically bundle the parser so we can require it in node without compiling the whole src dir + const tempFile = path.join(__dirname, 'temp_pipelineParser.js'); + await esbuild.build({ + entryPoints: [path.join(__dirname, '../../src/parsers/pipelineParser.ts')], + bundle: true, + outfile: tempFile, + format: 'cjs', + platform: 'node', + external: ['vscode', '*/componentService', '*/componentCacheManager'] + }); + + try { + const { PipelineParser } = originalRequire.apply(module, [tempFile]); + + console.log('\nTest 1: Stage merging works correctly and adds implicit stages'); + try { + const parser = new PipelineParser(10); + const yaml = ` +stages: + - custom1 + - custom2 + +job1: + stage: custom1 + script: echo "custom1" + +job2: + stage: custom2 + script: echo "custom2" + +job3: + stage: test + script: echo "implicit fallback" +`; + const graph = await parser.parse(yaml, 'test.yml'); + const stageNames = graph.stages.map(s => s.name); + + // .pre and .post always bookend; test is implicit and appended after custom stages + assert.ok(stageNames.includes('.pre'), '.pre must be present'); + assert.ok(stageNames.includes('.post'), '.post must be present'); + assert.ok(stageNames.includes('custom1'), 'custom1 must be present'); + assert.ok(stageNames.includes('custom2'), 'custom2 must be present'); + assert.ok(stageNames.includes('test'), 'implicit test stage must be present'); + // .pre must come before custom stages; .post must come last + assert.ok(stageNames.indexOf('.pre') < stageNames.indexOf('custom1'), '.pre before custom1'); + assert.ok(stageNames.indexOf('.post') === stageNames.length - 1, '.post must be last'); + + const testStage = graph.stages.find(s => s.name === 'test'); + assert.ok(testStage, 'test stage must exist'); + assert.strictEqual(testStage.jobs.length, 1, 'test stage must have 1 job'); + assert.strictEqual(testStage.jobs[0].name, 'job3', 'job3 must be in test stage'); + + console.log('Stage merging: PASS ✅'); + passed++; + } catch (e) { + console.error('Stage merging: FAIL ❌', e.message); + failed++; + } + + console.log('\nTest 2: Circular include detection — records error and does not crash'); + try { + const parser = new PipelineParser(10); + // This remote URL will fail to fetch (no real network in tests). + // The parser should record an error and still return a valid graph. + const yaml = `include:\n - remote: https://example.com/remote-a.yml\nbaseJob:\n script: echo "base"`; + const graph = await parser.parse(yaml, 'base.yml'); + + // baseJob must always be extracted regardless of include failures + const jobs = graph.stages.flatMap(s => s.jobs).map(j => j.name); + assert.ok(jobs.includes('baseJob'), 'baseJob should always be present'); + + // The graph must be returned (no crash), and stages must be well-formed + assert.ok(Array.isArray(graph.stages), 'stages must be an array'); + assert.ok(Array.isArray(graph.errors), 'errors must be an array'); + + console.log('Resilient error handling: PASS ✅'); + passed++; + } catch (e) { + console.error('Resilient error handling: FAIL ❌', e.message); + failed++; + } + + console.log('\nTest 3: Variable interpolation in remote includes'); + try { + const parser = new PipelineParser(10); + // $CI_SERVER_FQDN should be replaced with the gitlabInstance from context. + // The fetch will fail (no network), but we can verify the error message contains + // the interpolated URL (not the raw $CI_SERVER_FQDN placeholder). + const yaml = `include:\n - remote: https://$CI_SERVER_FQDN/my-remote.yml`; + + const graph = await parser.parse(yaml, 'base.yml', { + gitlabInstance: 'gitlab.custom.com', + serverUrl: 'https://gitlab.custom.com' + }); + + // The graph must be returned without crashing + assert.ok(Array.isArray(graph.stages), 'stages must be returned'); + + // Any error message should reference the interpolated domain, not the raw variable + const errorText = graph.errors.join(' '); + assert.ok( + !errorText.includes('$CI_SERVER_FQDN'), + `Error should not contain un-interpolated variable. Got: ${errorText}` + ); + + console.log('Variable interpolation: PASS ✅'); + passed++; + } catch (e) { + console.error('Variable interpolation: FAIL ❌', e.message); + failed++; + } + + + console.log('\nTest 4: Pipeline Execution Policy (PEP) documents are parsed correctly'); + try { + const parser = new PipelineParser(10); + const pepYaml = ` +pipeline_execution_policy: + - name: SAST Policy + enabled: true + pipeline: + stages: + - security_test + sast_job: + stage: security_test + script: echo "running SAST" + - name: Secret Detection Policy + enabled: true + pipeline: + stages: + - secret_scan + secret_detection_job: + stage: secret_scan + script: echo "running secret detection" +`; + const graph = await parser.parse(pepYaml, 'security-policies.yml'); + const stageNames = graph.stages.map(s => s.name); + const jobNames = graph.stages.flatMap(s => s.jobs).map(j => j.name); + + // The top-level pipeline_execution_policy key must NOT appear as a job + assert.ok(!jobNames.includes('pipeline_execution_policy'), + 'pipeline_execution_policy must not be treated as a job'); + + // Stages from both policies must be present + assert.ok(stageNames.includes('security_test'), 'security_test stage must be present'); + assert.ok(stageNames.includes('secret_scan'), 'secret_scan stage must be present'); + + // Jobs from both policies must be extracted + assert.ok(jobNames.includes('sast_job'), 'sast_job must be extracted from SAST policy'); + assert.ok(jobNames.includes('secret_detection_job'), 'secret_detection_job must be extracted from Secret Detection policy'); + + // Job sources should identify the policy they came from + const sastJob = graph.stages.flatMap(s => s.jobs).find(j => j.name === 'sast_job'); + assert.ok(sastJob?.source.includes('SAST Policy'), `sast_job source should reference policy name, got: ${sastJob?.source}`); + + console.log('PEP document parsing: PASS ✅'); + passed++; + } catch (e) { + console.error('PEP document parsing: FAIL ❌', e.message); + failed++; + } + + console.log('\nTest 5: alwaysInclude with absolute PEP path does not corrupt the main pipeline YAML'); + // Run WITHOUT a workspace folder so the dir-relative fallback is exercised. + // (The pep-policy.gitlab-ci.yml includes pipelines/secret-detection.gitlab-ci.yml + // which sits next to it; without a workspace the parser must find it via dirname.) + vscodeMock.workspace.workspaceFolders = []; + try { + const complexFixture = path.join(__dirname, '../../tests/fixtures/complex-pipeline.gitlab-ci.yml'); + const pepFixture = path.join(__dirname, '../../tests/fixtures/pep-policy.gitlab-ci.yml'); + + const complexContent = fs.readFileSync(complexFixture, 'utf8'); + + // The key scenario: PEP file passed as extraInclude (what alwaysInclude does). + // Previously this was injected as a YAML string causing duplicate-key corruption. + const extraIncludes = [{ local: pepFixture }]; + + const parser = new PipelineParser(10); + const graph = await parser.parse(complexContent, complexFixture, {}, extraIncludes); + + // 1. The main file must parse cleanly — no YAML corruption errors + const parseErrors = graph.errors.filter(e => e.includes('Failed to parse YAML')); + assert.strictEqual(parseErrors.length, 0, + `YAML parse errors must be zero. Got: ${parseErrors.join('; ')}`); + + const allJobs = graph.stages.flatMap(s => s.jobs); + const jobNames = allJobs.map(j => j.name); + + // 2. Jobs from the complex pipeline main file must be present + assert.ok(jobNames.includes('main_app_lint'), 'main_app_lint must be present from complex pipeline'); + assert.ok(jobNames.includes('main_app_deploy'), 'main_app_deploy must be present from complex pipeline'); + + // 3. Inline PEP job (SAST policy) must be present + assert.ok(jobNames.includes('sast_job'), 'sast_job must be extracted from inline SAST policy'); + + // 4. Job from the nested local include (Secret Detection policy → pipelines/secret-detection.gitlab-ci.yml) + // This tests the dir-relative fallback resolution path. + assert.ok(jobNames.includes('secret_detection_job'), + 'secret_detection_job must be resolved from PEP nested local include'); + + // 5. pipeline_execution_policy must NOT appear as a job + assert.ok(!jobNames.includes('pipeline_execution_policy'), + 'pipeline_execution_policy must not be treated as a job'); + + // 6. The nested include file must appear in includedSources + const includedPaths = graph.includedSources; + const nestedIncluded = includedPaths.some(s => s.includes('secret-detection.gitlab-ci.yml')); + assert.ok(nestedIncluded, + `Nested PEP include must appear in includedSources. Got: ${includedPaths.join(', ')}`); + + console.log('alwaysInclude + complex pipeline: PASS ✅'); + passed++; + } catch (e) { + console.error('alwaysInclude + complex pipeline: FAIL ❌', e.message); + failed++; + } + + console.log('\nTest 6: Windows absolute paths are not misidentified as project shorthands'); + try { + const parser = new PipelineParser(10); + // Mock a Windows-style absolute path include + const windowsPath = 'C:\\dev\\project\\ci-template.yml'; + + // We use a mock fs.readFileSync inside resolveLocalInclude indirectly + // But the primary check is whether it calls resolveLocalInclude vs hitting the project branch. + // If it hits the project branch, it will try to fetch via API and fail with a specific error. + + // Let's use a non-existent absolute path and check the error message. + // It should say "Cannot find local file" (local branch) + // NOT "Could not fetch project file" (project branch). + const graph = await parser.parse('stages: [test]', 'main.yml', {}, [{ local: windowsPath }]); + + const hasProjectError = graph.errors.some(e => e.includes('Could not fetch project file')); + const hasLocalError = graph.errors.some(e => e.includes('Cannot find local file')); + + assert.strictEqual(hasProjectError, false, 'Should not be identified as a project include'); + assert.strictEqual(hasLocalError, true, 'Should be identified as a (missing) local include'); + + console.log('Windows path handling: PASS ✅'); + passed++; + } catch (e) { + console.error('Windows path handling: FAIL ❌', e.message); + failed++; + } + + console.log('\nTest 7: Include tree captures hierarchical relationships'); + try { + const parser = new PipelineParser(10); + // Using the mocked remote-a.yml (which includes remote-b.yml) + const yaml = `include:\n - remote: https://example.com/remote-a.yml`; + const graph = await parser.parse(yaml, 'main.yml'); + + // Expected Structure: + // main.yml + // └── remote-a.yml + // └── remote-b.yml + + assert.strictEqual(graph.errors.length, 0, `Should have no errors, got: ${graph.errors.join('; ')}`); + assert.strictEqual(graph.includeTree.name, 'main.yml'); + assert.strictEqual(graph.includeTree.children.length, 1); + const aNode = graph.includeTree.children[0]; + assert.ok(aNode.name.includes('remote-a.yml'), 'aNode name should contain remote-a.yml'); + assert.strictEqual(aNode.children.length, 1, 'aNode should have 1 child (remote-b.yml)'); + const bNode = aNode.children[0]; + assert.ok(bNode.name.includes('remote-b.yml'), 'bNode name should contain remote-b.yml'); + + console.log('Include tree: PASS ✅'); + passed++; + } catch (e) { + console.error('Include tree: FAIL ❌', e.message); + failed++; + } + } finally { + if (fs.existsSync(tempFile)) { + fs.unlinkSync(tempFile); + } + } + + + console.log(`\n=== Pipeline Parser Test Summary ===`); + console.log(`Total tests: 7`); + console.log(`Passed: ${passed} ✅`); + console.log(`Failed: ${failed} ${failed > 0 ? '❌' : ''}`); + + if (failed > 0) { + process.exit(1); + } +} + +runTests().catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/tests/unit/url-parsing.test.js b/tests/unit/url-parsing.test.js new file mode 100644 index 00000000..7bc07517 --- /dev/null +++ b/tests/unit/url-parsing.test.js @@ -0,0 +1,123 @@ +/** + * URL Parsing Tests + * + * Tests the URL parsing logic used in componentDetector.ts + * to ensure GitLab component URLs are correctly parsed into + * project path, component name, and version. + */ + +console.log('=== URL Parsing Tests ==='); + +/** + * Test the URL parsing logic that extracts project path, component name, and version + * from GitLab component URLs + */ +function testUrlParsing() { + const testCases = [ + { + name: 'GitLab.com component URL', + url: 'https://gitlab.com/components/opentofu/full-pipeline@2.9.0', + expected: { + gitlabInstance: 'gitlab.com', + projectPath: 'components/opentofu', + componentName: 'full-pipeline', + version: '2.9.0' + } + }, + { + name: 'Component URL without version', + url: 'https://gitlab.example.com/group/project/my-component', + expected: { + gitlabInstance: 'gitlab.example.com', + projectPath: 'group/project', + componentName: 'my-component', + version: undefined + } + }, + { + name: 'Simple component URL', + url: 'https://gitlab.com/user/simple-component@latest', + expected: { + gitlabInstance: 'gitlab.com', + projectPath: 'user', + componentName: 'simple-component', + version: 'latest' + } + } + ]; + + let passed = 0; + let failed = 0; + + testCases.forEach((testCase, index) => { + console.log(`\nTest ${index + 1}: ${testCase.name}`); + console.log(`URL: ${testCase.url}`); + + // This is the same parsing logic used in componentDetector.ts + let projectPath, version, componentName, gitlabInstance; + + if (testCase.url.includes('@')) { + const urlParts = testCase.url.split('@'); + const baseUrl = urlParts[0]; + version = urlParts[1]; + + const baseUrlObj = new URL(baseUrl); + const fullPath = baseUrlObj.pathname.substring(1); + const pathParts = fullPath.split('/'); + componentName = pathParts.pop() || ''; + projectPath = pathParts.join('/'); + gitlabInstance = baseUrlObj.hostname; + } else { + const url = new URL(testCase.url); + const fullPath = url.pathname.substring(1); + const pathParts = fullPath.split('/'); + componentName = pathParts.pop() || ''; + projectPath = pathParts.join('/'); + gitlabInstance = url.hostname; + version = undefined; + } + + const actual = { gitlabInstance, projectPath, componentName, version }; + + // Check if results match expected + const isCorrect = + actual.gitlabInstance === testCase.expected.gitlabInstance && + actual.projectPath === testCase.expected.projectPath && + actual.componentName === testCase.expected.componentName && + actual.version === testCase.expected.version; + + console.log('Results:'); + console.log(` GitLab Instance: ${actual.gitlabInstance} (expected: ${testCase.expected.gitlabInstance})`); + console.log(` Project Path: ${actual.projectPath} (expected: ${testCase.expected.projectPath})`); + console.log(` Component Name: ${actual.componentName} (expected: ${testCase.expected.componentName})`); + console.log(` Version: ${actual.version} (expected: ${testCase.expected.version})`); + console.log(`Result: ${isCorrect ? 'PASS ✅' : 'FAIL ❌'}`); + + if (isCorrect) { + passed++; + } else { + failed++; + } + }); + + console.log(`\n=== URL Parsing Test Summary ===`); + console.log(`Total tests: ${testCases.length}`); + console.log(`Passed: ${passed} ✅`); + console.log(`Failed: ${failed} ${failed > 0 ? '❌' : ''}`); + console.log(`Success rate: ${Math.round((passed / testCases.length) * 100)}%`); + + return failed === 0; +} + +// Run the tests +const allTestsPassed = testUrlParsing(); + +if (allTestsPassed) { + console.log('\n🎉 All URL parsing tests passed!'); + // eslint-disable-next-line no-undef + process.exit(0); +} else { + console.log('\n💥 Some tests failed!'); + // eslint-disable-next-line no-undef + process.exit(1); +}