From 4966c860e51c810393c838d55d4bd5a4c68d6bc7 Mon Sep 17 00:00:00 2001 From: Sam Meluch Date: Wed, 1 Apr 2026 23:39:21 +0000 Subject: [PATCH 1/5] feat: add `azldev advanced ct-tools config-dump` command Add a new CLI command that parses Azure Linux distro configuration TOML files, recursively resolves includes via deep merge, expands koji-targets/build-roots/mock-options templates, filters to a specified Control Tower environment, and outputs the fully resolved config as JSON or YAML. New packages/files: - internal/app/azldev/core/cttools: types, TOML loader with glob include resolution and cycle detection, template resolver - internal/app/azldev/cmds/advanced/cttools.go: CLI wiring Flags: --config (required), --environment (required), --format (json|yaml) --- docs/user/reference/cli/azldev_advanced.md | 1 + .../reference/cli/azldev_advanced_ct-tools.md | 40 +++ .../azldev_advanced_ct-tools_config-dump.md | 56 ++++ internal/app/azldev/cmds/advanced/advanced.go | 1 + internal/app/azldev/cmds/advanced/cttools.go | 119 ++++++++ .../app/azldev/cmds/advanced/cttools_test.go | 41 +++ internal/app/azldev/core/cttools/loader.go | 171 +++++++++++ .../app/azldev/core/cttools/loader_test.go | 207 +++++++++++++ internal/app/azldev/core/cttools/resolver.go | 187 ++++++++++++ .../app/azldev/core/cttools/resolver_test.go | 275 ++++++++++++++++++ internal/app/azldev/core/cttools/types.go | 116 ++++++++ scenario/clismoke_test.go | 92 ++++++ scenario/testdata/cttools/common.toml | 4 + scenario/testdata/cttools/distro.toml | 7 + .../testdata/cttools/resources/ct-test.toml | 12 + .../cttools/templates/koji-targets.toml | 39 +++ .../testdata/cttools/templates/options.toml | 12 + scenario/testdata/cttools/versions/v1.toml | 30 ++ 18 files changed, 1410 insertions(+) create mode 100644 docs/user/reference/cli/azldev_advanced_ct-tools.md create mode 100644 docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md create mode 100644 internal/app/azldev/cmds/advanced/cttools.go create mode 100644 internal/app/azldev/cmds/advanced/cttools_test.go create mode 100644 internal/app/azldev/core/cttools/loader.go create mode 100644 internal/app/azldev/core/cttools/loader_test.go create mode 100644 internal/app/azldev/core/cttools/resolver.go create mode 100644 internal/app/azldev/core/cttools/resolver_test.go create mode 100644 internal/app/azldev/core/cttools/types.go create mode 100644 scenario/testdata/cttools/common.toml create mode 100644 scenario/testdata/cttools/distro.toml create mode 100644 scenario/testdata/cttools/resources/ct-test.toml create mode 100644 scenario/testdata/cttools/templates/koji-targets.toml create mode 100644 scenario/testdata/cttools/templates/options.toml create mode 100644 scenario/testdata/cttools/versions/v1.toml diff --git a/docs/user/reference/cli/azldev_advanced.md b/docs/user/reference/cli/azldev_advanced.md index 62bac139..b6a5141c 100644 --- a/docs/user/reference/cli/azldev_advanced.md +++ b/docs/user/reference/cli/azldev_advanced.md @@ -37,6 +37,7 @@ output but fully supported. ### SEE ALSO * [azldev](azldev.md) - 🐧 Azure Linux Dev Tool +* [azldev advanced ct-tools](azldev_advanced_ct-tools.md) - Control Tower tools * [azldev advanced mcp](azldev_advanced_mcp.md) - Run in MCP server mode * [azldev advanced mock](azldev_advanced_mock.md) - Run RPM mock tool * [azldev advanced wget](azldev_advanced_wget.md) - Download files via https diff --git a/docs/user/reference/cli/azldev_advanced_ct-tools.md b/docs/user/reference/cli/azldev_advanced_ct-tools.md new file mode 100644 index 00000000..54263745 --- /dev/null +++ b/docs/user/reference/cli/azldev_advanced_ct-tools.md @@ -0,0 +1,40 @@ + + +## azldev advanced ct-tools + +Control Tower tools + +### Synopsis + +Control Tower tools for working with distro configuration. + +Provides utilities for parsing, resolving, and dumping the fully merged +distro configuration used by Control Tower environments. + +### Options + +``` + -h, --help help for ct-tools +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev advanced](azldev_advanced.md) - Advanced operations +* [azldev advanced ct-tools config-dump](azldev_advanced_ct-tools_config-dump.md) - Dump fully resolved distro config + diff --git a/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md b/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md new file mode 100644 index 00000000..063fd38a --- /dev/null +++ b/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md @@ -0,0 +1,56 @@ + + +## azldev advanced ct-tools config-dump + +Dump fully resolved distro config + +### Synopsis + +Parse and resolve all distro configuration TOML files starting from a +top-level file, merge includes, expand all templates (koji-targets, +build-roots, mock-options), and output the fully resolved configuration +filtered to a specific Control Tower environment. + +``` +azldev advanced ct-tools config-dump [flags] +``` + +### Examples + +``` + # Dump config for ct-dev as JSON + azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-dev + + # Dump config for ct-prod as YAML + azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-prod --format yaml +``` + +### Options + +``` + --config string Path to the top-level TOML configuration file + --environment string Control Tower environment name (e.g. ct-dev, ct-staging, ct-prod) + --format string Output format: json or yaml (default "json") + -h, --help help for config-dump +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev advanced ct-tools](azldev_advanced_ct-tools.md) - Control Tower tools + diff --git a/internal/app/azldev/cmds/advanced/advanced.go b/internal/app/azldev/cmds/advanced/advanced.go index a52741d8..e8c3a8be 100644 --- a/internal/app/azldev/cmds/advanced/advanced.go +++ b/internal/app/azldev/cmds/advanced/advanced.go @@ -23,6 +23,7 @@ output but fully supported.`, } app.AddTopLevelCommand(cmd) + ctToolsOnAppInit(app, cmd) mcpOnAppInit(app, cmd) mockOnAppInit(app, cmd) wgetOnAppInit(app, cmd) diff --git a/internal/app/azldev/cmds/advanced/cttools.go b/internal/app/azldev/cmds/advanced/cttools.go new file mode 100644 index 00000000..d681c0aa --- /dev/null +++ b/internal/app/azldev/cmds/advanced/cttools.go @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package advanced + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +func ctToolsOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewCTToolsCmd()) +} + +// Constructs a [cobra.Command] for the "ct-tools" subcommand hierarchy. +func NewCTToolsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "ct-tools", + Short: "Control Tower tools", + Long: `Control Tower tools for working with distro configuration. + +Provides utilities for parsing, resolving, and dumping the fully merged +distro configuration used by Control Tower environments.`, + } + + cmd.AddCommand(NewConfigDumpCmd()) + + return cmd +} + +// Options controlling the config-dump command. +type ConfigDumpOptions struct { + // Path to the top-level TOML configuration file. + ConfigPath string + // The Control Tower environment to filter for (e.g. "ct-dev"). + Environment string + // Output format: "json" or "yaml". + Format string +} + +// Constructs a [cobra.Command] for the "ct-tools config-dump" subcommand. +func NewConfigDumpCmd() *cobra.Command { + options := &ConfigDumpOptions{} + + cmd := &cobra.Command{ + Use: "config-dump", + Short: "Dump fully resolved distro config", + Long: `Parse and resolve all distro configuration TOML files starting from a +top-level file, merge includes, expand all templates (koji-targets, +build-roots, mock-options), and output the fully resolved configuration +filtered to a specific Control Tower environment.`, + Example: ` # Dump config for ct-dev as JSON + azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-dev + + # Dump config for ct-prod as YAML + azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-prod --format yaml`, + RunE: azldev.RunFuncWithoutRequiredConfig(func(env *azldev.Env) (results interface{}, err error) { + return nil, RunConfigDump(options) + }), + } + + cmd.Flags().StringVar(&options.ConfigPath, "config", "", "Path to the top-level TOML configuration file") + + envHelp := "Control Tower environment name " + + "(e.g. ct-dev, ct-staging, ct-prod)" + cmd.Flags().StringVar(&options.Environment, "environment", "", envHelp) + cmd.Flags().StringVar(&options.Format, "format", "json", "Output format: json or yaml") + + _ = cmd.MarkFlagRequired("config") + _ = cmd.MarkFlagRequired("environment") + + return cmd +} + +// RunConfigDump loads, resolves, filters, and outputs the distro configuration. +func RunConfigDump(options *ConfigDumpOptions) error { + config, err := cttools.LoadConfig(options.ConfigPath) + if err != nil { + return fmt.Errorf("failed to load config from %#q:\n%w", options.ConfigPath, err) + } + + if err := cttools.ResolveTemplates(config); err != nil { + return fmt.Errorf("failed to resolve templates:\n%w", err) + } + + if err := cttools.FilterEnvironment(config, options.Environment); err != nil { + return fmt.Errorf("failed to filter environment:\n%w", err) + } + + var output []byte + + switch options.Format { + case "json": + output, err = json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal config to JSON:\n%w", err) + } + case "yaml": + output, err = yaml.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal config to YAML:\n%w", err) + } + default: + return fmt.Errorf("unsupported output format %#q; use 'json' or 'yaml'", options.Format) + } + + _, err = fmt.Fprintln(os.Stdout, string(output)) + if err != nil { + return fmt.Errorf("failed to write output:\n%w", err) + } + + return nil +} diff --git a/internal/app/azldev/cmds/advanced/cttools_test.go b/internal/app/azldev/cmds/advanced/cttools_test.go new file mode 100644 index 00000000..4e7a6855 --- /dev/null +++ b/internal/app/azldev/cmds/advanced/cttools_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package advanced_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/advanced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewCTToolsCmd(t *testing.T) { + cmd := advanced.NewCTToolsCmd() + require.NotNil(t, cmd) + assert.Equal(t, "ct-tools", cmd.Use) +} + +func TestNewConfigDumpCmd(t *testing.T) { + cmd := advanced.NewConfigDumpCmd() + require.NotNil(t, cmd) + assert.Equal(t, "config-dump", cmd.Use) +} + +func TestCTToolsCmd_HasConfigDumpSubcommand(t *testing.T) { + cmd := advanced.NewCTToolsCmd() + + subCmds := cmd.Commands() + found := false + + for _, sub := range subCmds { + if sub.Use == "config-dump" { + found = true + + break + } + } + + assert.True(t, found, "ct-tools should have config-dump subcommand") +} diff --git a/internal/app/azldev/core/cttools/loader.go b/internal/app/azldev/core/cttools/loader.go new file mode 100644 index 00000000..b8c47045 --- /dev/null +++ b/internal/app/azldev/core/cttools/loader.go @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/pelletier/go-toml/v2" +) + +// LoadConfig loads a distro config starting from the given top-level TOML file path. +// It recursively resolves `include` directives (relative glob paths), deep-merges +// all included files, and returns the final merged raw map. +func LoadConfig(topLevelPath string) (*DistroConfig, error) { + absPath, err := filepath.Abs(topLevelPath) + if err != nil { + return nil, fmt.Errorf("failed to resolve absolute path for %#q:\n%w", topLevelPath, err) + } + + visited := make(map[string]bool) + + merged, err := loadAndMerge(absPath, visited) + if err != nil { + return nil, err + } + + // Remove the include key from the merged map before marshalling to typed struct. + delete(merged, "include") + + // Re-serialize the merged map to TOML, then unmarshal into the typed struct. + buf, err := toml.Marshal(merged) + if err != nil { + return nil, fmt.Errorf("failed to marshal merged config:\n%w", err) + } + + var config DistroConfig + if err := toml.Unmarshal(buf, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal merged config into typed struct:\n%w", err) + } + + return &config, nil +} + +// loadAndMerge loads a single TOML file, processes its include directives, +// and returns the deep-merged result as a raw map. +func loadAndMerge(absPath string, visited map[string]bool) (map[string]any, error) { + if visited[absPath] { + return nil, fmt.Errorf("circular include detected for %#q", absPath) + } + + visited[absPath] = true + + data, err := os.ReadFile(absPath) + if err != nil { + return nil, fmt.Errorf("failed to read %#q:\n%w", absPath, err) + } + + var raw map[string]any + if err := toml.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("failed to parse TOML %#q:\n%w", absPath, err) + } + + // Extract and process includes. + includes, err := extractIncludes(raw, absPath) + if err != nil { + return nil, err + } + + // Start with the current file's data (without include key). + result := make(map[string]any) + deepMergeMaps(result, raw) + delete(result, "include") + + // Load each included file and merge into result. + dir := filepath.Dir(absPath) + + for _, pattern := range includes { + globPath := filepath.Join(dir, pattern) + + matches, err := filepath.Glob(globPath) + if err != nil { + return nil, fmt.Errorf("failed to glob %#q (from include in %#q):\n%w", globPath, absPath, err) + } + + for _, match := range matches { + matchAbs, err := filepath.Abs(match) + if err != nil { + return nil, fmt.Errorf("failed to resolve absolute path for %#q:\n%w", match, err) + } + + // Skip self-includes (e.g., when a glob like "./*.toml" matches the current file). + if matchAbs == absPath { + continue + } + + child, err := loadAndMerge(matchAbs, visited) + if err != nil { + return nil, fmt.Errorf("error loading include %#q from %#q:\n%w", match, absPath, err) + } + + deepMergeMaps(result, child) + } + } + + return result, nil +} + +// extractIncludes reads the "include" key from a raw TOML map and returns it as a string slice. +func extractIncludes(raw map[string]any, filePath string) ([]string, error) { + includeVal, hasInclude := raw["include"] + if !hasInclude { + return nil, nil + } + + includeSlice, isSlice := includeVal.([]any) + if !isSlice { + return nil, fmt.Errorf("'include' in %#q must be an array of strings", filePath) + } + + result := make([]string, 0, len(includeSlice)) + + for _, v := range includeSlice { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("'include' entry in %#q must be a string, got %T", filePath, v) + } + + result = append(result, s) + } + + return result, nil +} + +// deepMergeMaps merges src into dst recursively. For map values, sub-maps are merged recursively. +// For slice values, slices are concatenated. For all other types, src overwrites dst. +func deepMergeMaps(dst, src map[string]any) { + for key, srcVal := range src { + dstVal, exists := dst[key] + if !exists { + dst[key] = srcVal + + continue + } + + // If both are maps, merge recursively. + srcMap, srcIsMap := srcVal.(map[string]any) + dstMap, dstIsMap := dstVal.(map[string]any) + + if srcIsMap && dstIsMap { + deepMergeMaps(dstMap, srcMap) + + continue + } + + // If both are slices, concatenate. + srcSlice, srcIsSlice := srcVal.([]any) + dstSlice, dstIsSlice := dstVal.([]any) + + if srcIsSlice && dstIsSlice { + dst[key] = append(dstSlice, srcSlice...) + + continue + } + + // Otherwise, src overwrites dst. + dst[key] = srcVal + } +} diff --git a/internal/app/azldev/core/cttools/loader_test.go b/internal/app/azldev/core/cttools/loader_test.go new file mode 100644 index 00000000..377c5638 --- /dev/null +++ b/internal/app/azldev/core/cttools/loader_test.go @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadConfig_SimpleFile(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "main.toml"), ` +[distros.testdistro] +description = "Test Distro" +`) + + config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.NoError(t, err) + require.Contains(t, config.Distros, "testdistro") + assert.Equal(t, "Test Distro", config.Distros["testdistro"].Description) +} + +func TestLoadConfig_IncludeResolution(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "main.toml"), ` +include = ["sub.toml"] + +[distros.testdistro] +description = "Test Distro" +`) + + writeFile(t, filepath.Join(dir, "sub.toml"), ` +[mock-options-templates.rpm] +options = ["opt1", "opt2"] +`) + + config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.NoError(t, err) + + require.Contains(t, config.Distros, "testdistro") + require.Contains(t, config.MockOptionsTemplates, "rpm") + assert.Equal(t, []string{"opt1", "opt2"}, config.MockOptionsTemplates["rpm"].Options) +} + +func TestLoadConfig_NestedIncludes(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub"), 0o755)) + + writeFile(t, filepath.Join(dir, "main.toml"), ` +include = ["sub/mid.toml"] + +[distros.d] +description = "D" +`) + + writeFile(t, filepath.Join(dir, "sub", "mid.toml"), ` +include = ["leaf.toml"] + +[mock-options-templates.rpm] +options = ["a"] +`) + + writeFile(t, filepath.Join(dir, "sub", "leaf.toml"), ` +[build-root-templates.srpm] +packages = ["bash"] +`) + + config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.NoError(t, err) + + require.Contains(t, config.Distros, "d") + require.Contains(t, config.MockOptionsTemplates, "rpm") + require.Contains(t, config.BuildRootTemplates, "srpm") + assert.Equal(t, []string{"bash"}, config.BuildRootTemplates["srpm"].Packages) +} + +func TestLoadConfig_GlobIncludes(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "templates"), 0o755)) + + writeFile(t, filepath.Join(dir, "main.toml"), ` +include = ["templates/*.toml"] +`) + + writeFile(t, filepath.Join(dir, "templates", "mock.toml"), ` +[mock-options-templates.rpm] +options = ["opt1"] +`) + + writeFile(t, filepath.Join(dir, "templates", "build.toml"), ` +[build-root-templates.srpm] +packages = ["bash"] +`) + + config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.NoError(t, err) + + require.Contains(t, config.MockOptionsTemplates, "rpm") + require.Contains(t, config.BuildRootTemplates, "srpm") +} + +func TestLoadConfig_DeepMerge_MapsMerge(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "main.toml"), ` +include = ["extra.toml"] + +[distros.d1] +description = "D1" +`) + + writeFile(t, filepath.Join(dir, "extra.toml"), ` +[distros.d2] +description = "D2" +`) + + config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.NoError(t, err) + + require.Contains(t, config.Distros, "d1") + require.Contains(t, config.Distros, "d2") +} + +func TestLoadConfig_DeepMerge_ArraysConcatenate(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "main.toml"), ` +include = ["extra.toml"] + +[[distros.d.shadow-allowlists]] +tag-name = "tag1" +`) + + writeFile(t, filepath.Join(dir, "extra.toml"), ` +[[distros.d.shadow-allowlists]] +tag-name = "tag2" +`) + + config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.NoError(t, err) + + require.Contains(t, config.Distros, "d") + + allowlists := config.Distros["d"].ShadowAllowlists + require.Len(t, allowlists, 2) + assert.Equal(t, "tag1", allowlists[0].TagName) + assert.Equal(t, "tag2", allowlists[1].TagName) +} + +func TestLoadConfig_CircularInclude(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "a.toml"), `include = ["b.toml"]`) + writeFile(t, filepath.Join(dir, "b.toml"), `include = ["a.toml"]`) + + _, err := cttools.LoadConfig(filepath.Join(dir, "a.toml")) + require.Error(t, err) + assert.Contains(t, err.Error(), "circular include") +} + +func TestLoadConfig_MissingInclude(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "main.toml"), `include = ["nonexistent.toml"]`) + + // Glob returns no matches for nonexistent files, so this should succeed with empty config. + config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.NoError(t, err) + assert.Empty(t, config.Distros) +} + +func TestLoadConfig_InvalidTOML(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "main.toml"), `this is not valid toml {{{`) + + _, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse TOML") +} + +func TestLoadConfig_InvalidIncludeType(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, "main.toml"), `include = 42`) + + _, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + require.Error(t, err) + assert.Contains(t, err.Error(), "must be an array") +} + +// writeFile is a test helper that writes content to a file, creating it if needed. +func writeFile(t *testing.T, path, content string) { + t.Helper() + + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} diff --git a/internal/app/azldev/core/cttools/resolver.go b/internal/app/azldev/core/cttools/resolver.go new file mode 100644 index 00000000..e99a94fc --- /dev/null +++ b/internal/app/azldev/core/cttools/resolver.go @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools + +import ( + "fmt" +) + +// ResolveTemplates resolves all koji target templates for every git-source-repo in every distro +// version. It expands build-root references, mock-options references, and applies +// environment-prefix / repo-prefix / parent-prefix to produce [ResolvedKojiTarget] entries. +func ResolveTemplates(config *DistroConfig) error { + for distroName, distro := range config.Distros { + for versionName, version := range distro.Versions { + for repoName, repos := range version.GitSourceRepos { + for i := range repos { + repo := &repos[i] + + if err := resolveRepoTargets(config, &version, repo); err != nil { + return fmt.Errorf("error resolving targets for distro %#q, version %#q, repo %#q:\n%w", + distroName, versionName, repoName, err) + } + } + + version.GitSourceRepos[repoName] = repos + } + + distro.Versions[versionName] = version + } + + config.Distros[distroName] = distro + } + + return nil +} + +// FilterEnvironment removes all environments from the config except the specified one. +func FilterEnvironment(config *DistroConfig, envName string) error { + env, ok := config.Environments[envName] + if !ok { + available := make([]string, 0, len(config.Environments)) + for k := range config.Environments { + available = append(available, k) + } + + return fmt.Errorf("environment %#q not found; available: %v", envName, available) + } + + config.Environments = map[string]Environment{ + envName: env, + } + + return nil +} + +// resolveRepoTargets resolves all koji targets for a single git-source-repo. +func resolveRepoTargets(config *DistroConfig, version *Version, repo *GitSourceRepo) error { + templateSetName := repo.KojiTargets + + templateSet, ok := config.KojiTargetsTemplates[templateSetName] + if !ok { + return fmt.Errorf("koji-targets-template %#q not found", templateSetName) + } + + envPrefix := version.EnvironmentPrefix + repoPrefix := repo.RepoPrefix + parentPrefix := repo.ParentPrefix + + var resolved []ResolvedKojiTarget + + for targetName, targets := range templateSet { + for _, tmpl := range targets { + rt, err := resolveOneTarget(config, version, envPrefix, repoPrefix, parentPrefix, targetName, &tmpl) + if err != nil { + return fmt.Errorf("error resolving target %#q:\n%w", targetName, err) + } + + resolved = append(resolved, *rt) + } + } + + repo.ResolvedKojiTargets = resolved + + return nil +} + +// resolveOneTarget resolves a single koji target template into a [ResolvedKojiTarget]. +func resolveOneTarget( + config *DistroConfig, + version *Version, + envPrefix, repoPrefix, parentPrefix, targetName string, + tmpl *KojiTarget, +) (*ResolvedKojiTarget, error) { + resolvedName := applyPrefix(envPrefix, repoPrefix, targetName) + resolvedOutputTag := applyPrefix(envPrefix, repoPrefix, tmpl.OutputTag) + + var resolvedParentTag string + if tmpl.ParentTag != "" { + resolvedParentTag = applyPrefix(parentPrefix, "", tmpl.ParentTag) + } + + // Resolve build roots. + buildRoots, err := resolveBuildRoots(config, tmpl.BuildRoots) + if err != nil { + return nil, err + } + + // Resolve mock options. + mockOpts, err := resolveMockOptions(config, tmpl.MockOptionsBase) + if err != nil { + return nil, err + } + + // Resolve mock-dist-tag (dereference field name on the version). + var mockDistTag string + if tmpl.MockDistTag != "" { + mockDistTag, err = resolveDistTag(version, tmpl.MockDistTag) + if err != nil { + return nil, err + } + } + + resolvedTarget := &ResolvedKojiTarget{ + Name: resolvedName, + OutputTag: resolvedOutputTag, + ParentTag: resolvedParentTag, + BuildRoots: buildRoots, + MockOptions: mockOpts, + MockDistTag: mockDistTag, + ExternalRepos: tmpl.ExternalRepos, + } + + return resolvedTarget, nil +} + +// applyPrefix constructs "{envPrefix}-{repoPrefix}-{suffix}" or "{envPrefix}-{suffix}" when +// repoPrefix is empty. +func applyPrefix(envPrefix, repoPrefix, suffix string) string { + if repoPrefix != "" { + return envPrefix + "-" + repoPrefix + "-" + suffix + } + + return envPrefix + "-" + suffix +} + +// resolveBuildRoots expands build-root references to their package lists. +func resolveBuildRoots(config *DistroConfig, refs []BuildRootRef) ([]ResolvedBuildRoot, error) { + resolved := make([]ResolvedBuildRoot, 0, len(refs)) + + for _, ref := range refs { + tmpl, ok := config.BuildRootTemplates[ref.Value] + if !ok { + return nil, fmt.Errorf("build-root-template %#q not found", ref.Value) + } + + resolved = append(resolved, ResolvedBuildRoot{ + Type: ref.Type, + Packages: tmpl.Packages, + }) + } + + return resolved, nil +} + +// resolveMockOptions looks up a mock-options template by name and returns its options. +func resolveMockOptions(config *DistroConfig, templateName string) ([]string, error) { + tmpl, ok := config.MockOptionsTemplates[templateName] + if !ok { + return nil, fmt.Errorf("mock-options-template %#q not found", templateName) + } + + return tmpl.Options, nil +} + +// resolveDistTag maps a mock-dist-tag field name (e.g., "rpm-macro-dist") to the corresponding +// value on the [Version]. +func resolveDistTag(version *Version, fieldName string) (string, error) { + switch fieldName { + case "rpm-macro-dist": + return version.RPMMacroDist, nil + case "rpm-macro-dist-bootstrap": + return version.RPMMacroDistBootstrap, nil + default: + return "", fmt.Errorf("unknown mock-dist-tag field %#q", fieldName) + } +} diff --git a/internal/app/azldev/core/cttools/resolver_test.go b/internal/app/azldev/core/cttools/resolver_test.go new file mode 100644 index 00000000..a56aceb6 --- /dev/null +++ b/internal/app/azldev/core/cttools/resolver_test.go @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveTemplates_BasicResolution(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "testdistro": { + Description: "Test", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "td1-dev", + RPMMacroDist: ".td1-dev", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "main": { + { + Ref: "https://example.com/repo.git", + DefaultBranch: "main", + DefaultKojiRPMTarget: "td1-dev-rpms-target", + KojiTargets: "base", + ParentPrefix: "td1-dev", + }, + }, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{ + "base": { + "rpms-target": { + { + OutputTag: "rpms-tag", + ParentTag: "bootstrap-rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + MockDistTag: "rpm-macro-dist", + }, + }, + }, + }, + MockOptionsTemplates: map[string]cttools.MockOptionsTemplate{ + "rpm": {Options: []string{"opt1", "opt2"}}, + }, + BuildRootTemplates: map[string]cttools.BuildRootTemplate{ + "rpm": {Packages: []string{"bash", "gcc"}}, + }, + } + + err := cttools.ResolveTemplates(config) + require.NoError(t, err) + + repos := config.Distros["testdistro"].Versions["1.0"].GitSourceRepos["main"] + require.Len(t, repos, 1) + + resolved := repos[0].ResolvedKojiTargets + require.Len(t, resolved, 1) + + target := resolved[0] + assert.Equal(t, "td1-dev-rpms-target", target.Name) + assert.Equal(t, "td1-dev-rpms-tag", target.OutputTag) + assert.Equal(t, "td1-dev-bootstrap-rpms-tag", target.ParentTag) + assert.Equal(t, []string{"opt1", "opt2"}, target.MockOptions) + assert.Equal(t, ".td1-dev", target.MockDistTag) + + require.Len(t, target.BuildRoots, 1) + assert.Equal(t, "build", target.BuildRoots[0].Type) + assert.Equal(t, []string{"bash", "gcc"}, target.BuildRoots[0].Packages) +} + +func TestResolveTemplates_WithRepoPrefix(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "d": { + Description: "D", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "azl4-dev", + RPMMacroDist: ".azl4-dev", + RPMMacroDistBootstrap: ".azl4-dev~bootstrap", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "nvidia": { + { + Ref: "https://example.com/nvidia.git", + DefaultBranch: "main", + DefaultKojiRPMTarget: "azl4-dev-nvidia-rpms-target", + KojiTargets: "prop", + RepoPrefix: "nvidia", + ParentPrefix: "azl4-dev", + }, + }, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{ + "prop": { + "bootstrap-rpms-target": { + { + OutputTag: "bootstrap-rpms-tag", + ParentTag: "bootstrap-rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + MockDistTag: "rpm-macro-dist-bootstrap", + }, + }, + "rpms-target": { + { + OutputTag: "rpms-tag", + ParentTag: "rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + MockDistTag: "rpm-macro-dist", + }, + }, + }, + }, + MockOptionsTemplates: map[string]cttools.MockOptionsTemplate{ + "rpm": {Options: []string{"opt1"}}, + }, + BuildRootTemplates: map[string]cttools.BuildRootTemplate{ + "rpm": {Packages: []string{"bash"}}, + }, + } + + err := cttools.ResolveTemplates(config) + require.NoError(t, err) + + repos := config.Distros["d"].Versions["1.0"].GitSourceRepos["nvidia"] + require.Len(t, repos, 1) + + resolved := repos[0].ResolvedKojiTargets + require.Len(t, resolved, 2) + + names := make(map[string]cttools.ResolvedKojiTarget, len(resolved)) + for _, rt := range resolved { + names[rt.Name] = rt + } + + bootstrap := names["azl4-dev-nvidia-bootstrap-rpms-target"] + assert.Equal(t, "azl4-dev-nvidia-bootstrap-rpms-tag", bootstrap.OutputTag) + assert.Equal(t, "azl4-dev-bootstrap-rpms-tag", bootstrap.ParentTag) + assert.Equal(t, ".azl4-dev~bootstrap", bootstrap.MockDistTag) + + rpms := names["azl4-dev-nvidia-rpms-target"] + assert.Equal(t, "azl4-dev-nvidia-rpms-tag", rpms.OutputTag) + assert.Equal(t, "azl4-dev-rpms-tag", rpms.ParentTag) + assert.Equal(t, ".azl4-dev", rpms.MockDistTag) +} + +func TestResolveTemplates_ExternalReposCopied(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "d": { + Description: "D", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "p", + RPMMacroDist: ".p", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "r": {{ + Ref: "https://example.com", + DefaultBranch: "main", + DefaultKojiRPMTarget: "p-rpms-target", + KojiTargets: "tmpl", + ParentPrefix: "p", + }}, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{ + "tmpl": { + "rpms-target": {{ + OutputTag: "rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + ExternalRepos: []cttools.ExternalRepo{ + {Name: "fedora", URL: "https://fedora.example.com", MergeMode: "bare"}, + }, + }}, + }, + }, + MockOptionsTemplates: map[string]cttools.MockOptionsTemplate{ + "rpm": {Options: []string{"opt1"}}, + }, + BuildRootTemplates: map[string]cttools.BuildRootTemplate{ + "rpm": {Packages: []string{"bash"}}, + }, + } + + err := cttools.ResolveTemplates(config) + require.NoError(t, err) + + resolved := config.Distros["d"].Versions["1.0"].GitSourceRepos["r"][0].ResolvedKojiTargets + require.Len(t, resolved, 1) + require.Len(t, resolved[0].ExternalRepos, 1) + assert.Equal(t, "fedora", resolved[0].ExternalRepos[0].Name) +} + +func TestResolveTemplates_MissingTemplate(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "d": { + Description: "D", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "p", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "r": {{ + Ref: "https://example.com", + DefaultBranch: "main", + DefaultKojiRPMTarget: "p-rpms-target", + KojiTargets: "nonexistent", + ParentPrefix: "p", + }}, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{}, + } + + err := cttools.ResolveTemplates(config) + require.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent") +} + +func TestFilterEnvironment_Found(t *testing.T) { + config := &cttools.DistroConfig{ + Environments: map[string]cttools.Environment{ + "ct-dev": {Resources: map[string][]map[string]any{"r1": {{"interface-type": "rpm-repo"}}}}, + "ct-prod": {Resources: map[string][]map[string]any{"r2": {{"interface-type": "image-gallery"}}}}, + }, + } + + err := cttools.FilterEnvironment(config, "ct-dev") + require.NoError(t, err) + + require.Len(t, config.Environments, 1) + require.Contains(t, config.Environments, "ct-dev") +} + +func TestFilterEnvironment_NotFound(t *testing.T) { + config := &cttools.DistroConfig{ + Environments: map[string]cttools.Environment{ + "ct-dev": {Resources: map[string][]map[string]any{}}, + }, + } + + err := cttools.FilterEnvironment(config, "ct-staging") + require.Error(t, err) + assert.Contains(t, err.Error(), "ct-staging") + assert.Contains(t, err.Error(), "not found") +} diff --git a/internal/app/azldev/core/cttools/types.go b/internal/app/azldev/core/cttools/types.go new file mode 100644 index 00000000..78d0a581 --- /dev/null +++ b/internal/app/azldev/core/cttools/types.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package cttools provides utilities for parsing and resolving Azure Linux distro configuration. +// +//nolint:tagliatelle // JSON output must use hyphenated keys to match the distro-config schema. +package cttools + +// DistroConfig is the top-level structure for the fully parsed/merged Azure Linux distro configuration. +type DistroConfig struct { + Distros map[string]Distro `toml:"distros" json:"distros,omitempty" yaml:"distros,omitempty"` + KojiTargetsTemplates map[string]map[string][]KojiTarget `toml:"koji-targets-templates" json:"koji-targets-templates,omitempty" yaml:"koji-targets-templates,omitempty"` + MockOptionsTemplates map[string]MockOptionsTemplate `toml:"mock-options-templates" json:"mock-options-templates,omitempty" yaml:"mock-options-templates,omitempty"` + BuildRootTemplates map[string]BuildRootTemplate `toml:"build-root-templates" json:"build-root-templates,omitempty" yaml:"build-root-templates,omitempty"` + Environments map[string]Environment `toml:"environments" json:"environments,omitempty" yaml:"environments,omitempty"` +} + +// Distro represents a distro definition (e.g. "azurelinux"). +type Distro struct { + Description string `toml:"description" json:"description" yaml:"description"` + ShadowAllowlists []ShadowAllowlist `toml:"shadow-allowlists" json:"shadow-allowlists,omitempty" yaml:"shadow-allowlists,omitempty"` + Versions map[string]Version `toml:"versions" json:"versions,omitempty" yaml:"versions,omitempty"` +} + +// ShadowAllowlist is a tag-name entry in a distro's shadow allowlist. +type ShadowAllowlist struct { + TagName string `toml:"tag-name" json:"tag-name" yaml:"tag-name"` +} + +// Version represents a distro version definition (e.g. "4.0-dev"). +type Version struct { + Description string `toml:"description" json:"description" yaml:"description"` + ReleaseVer string `toml:"release-ver" json:"release-ver" yaml:"release-ver"` + EnvironmentPrefix string `toml:"environment-prefix" json:"environment-prefix" yaml:"environment-prefix"` + RPMMacroDist string `toml:"rpm-macro-dist" json:"rpm-macro-dist" yaml:"rpm-macro-dist"` + RPMMacroDistBootstrap string `toml:"rpm-macro-dist-bootstrap" json:"rpm-macro-dist-bootstrap" yaml:"rpm-macro-dist-bootstrap"` + GitSourceRepos map[string][]GitSourceRepo `toml:"git-source-repos" json:"git-source-repos,omitempty" yaml:"git-source-repos,omitempty"` + BuildChannels map[string][]BuildChannel `toml:"build-channels" json:"build-channels,omitempty" yaml:"build-channels,omitempty"` + PublishChannels map[string][]PublishChannel `toml:"publish-channels" json:"publish-channels,omitempty" yaml:"publish-channels,omitempty"` +} + +// GitSourceRepo represents a git source repository definition within a distro version. +type GitSourceRepo struct { + Ref string `toml:"ref" json:"ref" yaml:"ref"` + DefaultBranch string `toml:"default-branch" json:"default-branch" yaml:"default-branch"` + DefaultKojiRPMTarget string `toml:"default-koji-rpms-target" json:"default-koji-rpms-target" yaml:"default-koji-rpms-target"` + KojiTargets string `toml:"koji-targets" json:"koji-targets" yaml:"koji-targets"` + RepoPrefix string `toml:"repo-prefix" json:"repo-prefix,omitempty" yaml:"repo-prefix,omitempty"` + ParentPrefix string `toml:"parent-prefix" json:"parent-prefix" yaml:"parent-prefix"` + ResolvedKojiTargets []ResolvedKojiTarget `toml:"resolved-koji-targets" json:"resolved-koji-targets,omitempty" yaml:"resolved-koji-targets,omitempty"` +} + +// ResolvedKojiTarget is a fully resolved koji target with all prefixes applied. +type ResolvedKojiTarget struct { + Name string `toml:"name" json:"name" yaml:"name"` + OutputTag string `toml:"output-tag" json:"output-tag" yaml:"output-tag"` + ParentTag string `toml:"parent-tag" json:"parent-tag,omitempty" yaml:"parent-tag,omitempty"` + BuildRoots []ResolvedBuildRoot `toml:"build-roots" json:"build-roots" yaml:"build-roots"` + MockOptions []string `toml:"mock-options" json:"mock-options" yaml:"mock-options"` + MockDistTag string `toml:"mock-dist-tag" json:"mock-dist-tag,omitempty" yaml:"mock-dist-tag,omitempty"` + ExternalRepos []ExternalRepo `toml:"external-repos" json:"external-repos,omitempty" yaml:"external-repos,omitempty"` +} + +// ResolvedBuildRoot is a build root entry with the template expanded to a package list. +type ResolvedBuildRoot struct { + Type string `toml:"type" json:"type" yaml:"type"` + Packages []string `toml:"packages" json:"packages" yaml:"packages"` +} + +// KojiTarget is a koji build target definition from a template. +type KojiTarget struct { + OutputTag string `toml:"output-tag" json:"output-tag" yaml:"output-tag"` + ParentTag string `toml:"parent-tag" json:"parent-tag,omitempty" yaml:"parent-tag,omitempty"` + BuildRoots []BuildRootRef `toml:"build-roots" json:"build-roots" yaml:"build-roots"` + MockOptionsBase string `toml:"mock-options-base" json:"mock-options-base" yaml:"mock-options-base"` + MockDistTag string `toml:"mock-dist-tag" json:"mock-dist-tag,omitempty" yaml:"mock-dist-tag,omitempty"` + ExternalRepos []ExternalRepo `toml:"external-repos" json:"external-repos,omitempty" yaml:"external-repos,omitempty"` +} + +// BuildRootRef references a build-root template by name. +type BuildRootRef struct { + Type string `toml:"type" json:"type" yaml:"type"` + Value string `toml:"value" json:"value" yaml:"value"` +} + +// ExternalRepo is an external repository definition on a koji target. +type ExternalRepo struct { + Name string `toml:"name" json:"name" yaml:"name"` + URL string `toml:"url" json:"url" yaml:"url"` + MergeMode string `toml:"merge-mode" json:"merge-mode" yaml:"merge-mode"` +} + +// MockOptionsTemplate defines a reusable set of mock/rpm options. +type MockOptionsTemplate struct { + Options []string `toml:"options" json:"options" yaml:"options"` +} + +// BuildRootTemplate defines a reusable package list for koji build roots. +type BuildRootTemplate struct { + Packages []string `toml:"packages" json:"packages" yaml:"packages"` +} + +// Environment is a Control Tower environment definition. +type Environment struct { + Resources map[string][]map[string]any `toml:"resources" json:"resources" yaml:"resources"` +} + +// BuildChannel specifies a koji target for routing builds. +type BuildChannel struct { + KojiTarget string `toml:"koji-target" json:"koji-target" yaml:"koji-target"` +} + +// PublishChannel specifies a resource for publishing artifacts. +type PublishChannel struct { + PublishResource string `toml:"publish-resource" json:"publish-resource" yaml:"publish-resource"` +} diff --git a/scenario/clismoke_test.go b/scenario/clismoke_test.go index 27764317..abeaa32c 100644 --- a/scenario/clismoke_test.go +++ b/scenario/clismoke_test.go @@ -6,6 +6,7 @@ package scenario_tests import ( + "encoding/json" "path/filepath" "strings" "testing" @@ -13,6 +14,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/scenario/internal/cmdtest" "github.com/microsoft/azure-linux-dev-tools/scenario/internal/snapshot" "github.com/microsoft/azure-linux-dev-tools/scenario/internal/testhelpers" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -163,3 +165,93 @@ excluded-paths = ['build/**', 'out/**'] snapshot.TestSnapshottableCmd(t, test) } + +// Tests that `azldev advanced ct-tools config-dump` parses, resolves, and outputs +// a fully merged distro configuration as valid JSON. +func TestCTToolsConfigDump(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + // Use the self-contained test config under scenario/testdata/cttools/. + configPath, err := filepath.Abs("testdata/cttools/distro.toml") + require.NoError(t, err) + + test := cmdtest.NewScenarioTest( + "advanced", "ct-tools", "config-dump", + "--config", configPath, + "--environment", "ct-test", + "--format", "json", + ).Locally() + + results, err := test.Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode, "stderr: %s", results.Stderr) + + // Parse the JSON output and verify structure. + var config map[string]any + require.NoError(t, json.Unmarshal([]byte(results.Stdout), &config)) + + // Verify top-level keys. + assert.Contains(t, config, "distros") + assert.Contains(t, config, "koji-targets-templates") + assert.Contains(t, config, "mock-options-templates") + assert.Contains(t, config, "build-root-templates") + assert.Contains(t, config, "environments") + + // Verify distro was loaded. + distros, ok := config["distros"].(map[string]any) + require.True(t, ok) + assert.Contains(t, distros, "testdistro") + + // Verify environment was filtered to ct-test only. + envs, ok := config["environments"].(map[string]any) + require.True(t, ok) + assert.Len(t, envs, 1) + assert.Contains(t, envs, "ct-test") + + // Verify template resolution produced resolved-koji-targets. + td := distros["testdistro"].(map[string]any) + versions := td["versions"].(map[string]any) + v1 := versions["1.0-dev"].(map[string]any) + repos := v1["git-source-repos"].(map[string]any) + mainRepos := repos["main"].([]any) + mainRepo := mainRepos[0].(map[string]any) + + resolved, ok := mainRepo["resolved-koji-targets"].([]any) + require.True(t, ok) + assert.NotEmpty(t, resolved, "resolved-koji-targets should not be empty") + + // Verify a resolved target has the expected prefix. + first := resolved[0].(map[string]any) + name, ok := first["name"].(string) + require.True(t, ok) + assert.Contains(t, name, "td1-dev-") +} + +// Tests that `azldev advanced ct-tools config-dump` outputs valid YAML. +func TestCTToolsConfigDumpYAML(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + configPath, err := filepath.Abs("testdata/cttools/distro.toml") + require.NoError(t, err) + + test := cmdtest.NewScenarioTest( + "advanced", "ct-tools", "config-dump", + "--config", configPath, + "--environment", "ct-test", + "--format", "yaml", + ).Locally() + + results, err := test.Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode, "stderr: %s", results.Stderr) + assert.Contains(t, results.Stdout, "testdistro") + assert.Contains(t, results.Stdout, "ct-test") +} diff --git a/scenario/testdata/cttools/common.toml b/scenario/testdata/cttools/common.toml new file mode 100644 index 00000000..4b3ff5ff --- /dev/null +++ b/scenario/testdata/cttools/common.toml @@ -0,0 +1,4 @@ +include = [ + "templates/*.toml", + "resources/*.toml", +] diff --git a/scenario/testdata/cttools/distro.toml b/scenario/testdata/cttools/distro.toml new file mode 100644 index 00000000..8649781e --- /dev/null +++ b/scenario/testdata/cttools/distro.toml @@ -0,0 +1,7 @@ +include = [ + "versions/v1.toml", + "common.toml", +] + +[distros.testdistro] +description = "Test Distro" diff --git a/scenario/testdata/cttools/resources/ct-test.toml b/scenario/testdata/cttools/resources/ct-test.toml new file mode 100644 index 00000000..f6e18aa3 --- /dev/null +++ b/scenario/testdata/cttools/resources/ct-test.toml @@ -0,0 +1,12 @@ +[[environments.'ct-test'.resources.'dev-rpms-base']] +interface-type = "rpm-repo" +backend = "azure-blobstore-repo" +base-uri = "" +container-name = "$releasever/dev/base" + +[[environments.'ct-test'.resources.'dev-images']] +interface-type = "image-gallery" +backend = "azure-compute-gallery" +resource-name = "testGallery" +staging-blobstore-uri = "https://test.blob.core.windows.net/" +vm-definition-suffix = "-dev" diff --git a/scenario/testdata/cttools/templates/koji-targets.toml b/scenario/testdata/cttools/templates/koji-targets.toml new file mode 100644 index 00000000..1529c09b --- /dev/null +++ b/scenario/testdata/cttools/templates/koji-targets.toml @@ -0,0 +1,39 @@ +[[koji-targets-templates.base.'bootstrap-rpms-target']] +output-tag = "bootstrap-rpms-tag" +build-roots = [ + {"type" = "srpm-build", "value" = "srpm"}, + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist-bootstrap" +external-repos = [ + {"name" = "fedora-external", "url" = "https://fedora.example.com/$arch/os/", "merge-mode" = "bare"}, +] + +[[koji-targets-templates.base.'rpms-target']] +output-tag = "rpms-tag" +parent-tag = "bootstrap-rpms-tag" +build-roots = [ + {"type" = "srpm-build", "value" = "srpm"}, + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist" + +[[koji-targets-templates.prop.'bootstrap-rpms-target']] +output-tag = "bootstrap-rpms-tag" +parent-tag = "bootstrap-rpms-tag" +build-roots = [ + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist-bootstrap" + +[[koji-targets-templates.prop.'rpms-target']] +output-tag = "rpms-tag" +parent-tag = "rpms-tag" +build-roots = [ + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist" diff --git a/scenario/testdata/cttools/templates/options.toml b/scenario/testdata/cttools/templates/options.toml new file mode 100644 index 00000000..0843996e --- /dev/null +++ b/scenario/testdata/cttools/templates/options.toml @@ -0,0 +1,12 @@ +[mock-options-templates.rpm] +options = [ + "mock.isolation=simple", + "mock.package_manager=dnf5", + "rpm.macro.vendor=Test Corp", +] + +[build-root-templates.srpm] +packages = ["bash", "rpm-build"] + +[build-root-templates.rpm] +packages = ["bash", "gcc", "make"] diff --git a/scenario/testdata/cttools/versions/v1.toml b/scenario/testdata/cttools/versions/v1.toml new file mode 100644 index 00000000..c02dc5e1 --- /dev/null +++ b/scenario/testdata/cttools/versions/v1.toml @@ -0,0 +1,30 @@ +[distros.testdistro.versions.'1.0-dev'] +description = "Test Distro 1.0-dev" +release-ver = "1.0" +environment-prefix = "td1-dev" +rpm-macro-dist = ".td1-dev" +rpm-macro-dist-bootstrap = ".td1-dev~bootstrap" + +[[distros.testdistro.versions.'1.0-dev'.git-source-repos.'main']] +ref = "https://example.com/main.git" +default-branch = "main" +default-koji-rpms-target = "td1-dev-rpms-target" +koji-targets = "base" +parent-prefix = "td1-dev" + +[[distros.testdistro.versions.'1.0-dev'.git-source-repos.'proprietary']] +ref = "https://example.com/prop.git" +default-branch = "main" +default-koji-rpms-target = "td1-dev-prop-rpms-target" +koji-targets = "prop" +repo-prefix = "prop" +parent-prefix = "td1-dev" + +[[distros.testdistro.versions.'1.0-dev'.build-channels.'default-rpm']] +koji-target = "td1-dev-rpms-target" + +[[distros.testdistro.versions.'1.0-dev'.build-channels.'prop-rpm']] +koji-target = "td1-dev-prop-rpms-target" + +[[distros.testdistro.versions.'1.0-dev'.publish-channels.'rpm-base']] +publish-resource = "dev-rpms-base" From fd80deebd39cac6a90faffaafda4907ca5bf4bba Mon Sep 17 00:00:00 2001 From: Sam Meluch Date: Thu, 2 Apr 2026 00:39:15 +0000 Subject: [PATCH 2/5] refactor: address PR review for ct-tools config-dump - Use opctx.FS abstraction in loader instead of raw os.ReadFile / filepath.Glob; thread fs parameter through LoadConfig and helpers - Rewrite loader_test.go to use testctx.NewCtx() + fileutils for in-memory filesystem instead of t.TempDir() / os.WriteFile - Return *DistroConfig from RunConfigDump and let the framework handle serialization via --output-format; drop custom --format flag and yaml dependency - Pass *azldev.Env to RunConfigDump; use env.FS() for filesystem access - Sort map keys before iteration and sort resolved koji targets by name for deterministic output across runs - Add slog.Debug calls when loading files and resolving includes - Rename --config to --ct-config to avoid ambiguity with global --config-file; add MarkFlagFilename for shell completion - Remove TestCTToolsConfigDumpYAML scenario test (YAML output dropped) --- internal/app/azldev/cmds/advanced/cttools.go | 59 ++----- internal/app/azldev/core/cttools/loader.go | 35 ++-- .../app/azldev/core/cttools/loader_test.go | 155 +++++++++++------- internal/app/azldev/core/cttools/resolver.go | 39 ++++- scenario/clismoke_test.go | 29 +--- 5 files changed, 169 insertions(+), 148 deletions(-) diff --git a/internal/app/azldev/cmds/advanced/cttools.go b/internal/app/azldev/cmds/advanced/cttools.go index d681c0aa..e2a4fe17 100644 --- a/internal/app/azldev/cmds/advanced/cttools.go +++ b/internal/app/azldev/cmds/advanced/cttools.go @@ -4,14 +4,11 @@ package advanced import ( - "encoding/json" "fmt" - "os" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" "github.com/spf13/cobra" - "gopkg.in/yaml.v3" ) func ctToolsOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { @@ -40,8 +37,6 @@ type ConfigDumpOptions struct { ConfigPath string // The Control Tower environment to filter for (e.g. "ct-dev"). Environment string - // Output format: "json" or "yaml". - Format string } // Constructs a [cobra.Command] for the "ct-tools config-dump" subcommand. @@ -56,64 +51,44 @@ top-level file, merge includes, expand all templates (koji-targets, build-roots, mock-options), and output the fully resolved configuration filtered to a specific Control Tower environment.`, Example: ` # Dump config for ct-dev as JSON - azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-dev - - # Dump config for ct-prod as YAML - azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-prod --format yaml`, + azldev advanced ct-tools config-dump \ + --ct-config /path/to/azurelinux.toml \ + --environment ct-dev -O json`, RunE: azldev.RunFuncWithoutRequiredConfig(func(env *azldev.Env) (results interface{}, err error) { - return nil, RunConfigDump(options) + return RunConfigDump(env, options) }), } - cmd.Flags().StringVar(&options.ConfigPath, "config", "", "Path to the top-level TOML configuration file") + cmd.Flags().StringVar( + &options.ConfigPath, "ct-config", "", + "Path to the top-level CT distro TOML configuration file", + ) envHelp := "Control Tower environment name " + "(e.g. ct-dev, ct-staging, ct-prod)" cmd.Flags().StringVar(&options.Environment, "environment", "", envHelp) - cmd.Flags().StringVar(&options.Format, "format", "json", "Output format: json or yaml") - _ = cmd.MarkFlagRequired("config") + _ = cmd.MarkFlagRequired("ct-config") _ = cmd.MarkFlagRequired("environment") + _ = cmd.MarkFlagFilename("ct-config", "toml") return cmd } -// RunConfigDump loads, resolves, filters, and outputs the distro configuration. -func RunConfigDump(options *ConfigDumpOptions) error { - config, err := cttools.LoadConfig(options.ConfigPath) +// RunConfigDump loads, resolves, filters, and returns the distro configuration. +func RunConfigDump(env *azldev.Env, options *ConfigDumpOptions) (*cttools.DistroConfig, error) { + config, err := cttools.LoadConfig(env.FS(), options.ConfigPath) if err != nil { - return fmt.Errorf("failed to load config from %#q:\n%w", options.ConfigPath, err) + return nil, fmt.Errorf("failed to load config from %#q:\n%w", options.ConfigPath, err) } if err := cttools.ResolveTemplates(config); err != nil { - return fmt.Errorf("failed to resolve templates:\n%w", err) + return nil, fmt.Errorf("failed to resolve templates:\n%w", err) } if err := cttools.FilterEnvironment(config, options.Environment); err != nil { - return fmt.Errorf("failed to filter environment:\n%w", err) - } - - var output []byte - - switch options.Format { - case "json": - output, err = json.MarshalIndent(config, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal config to JSON:\n%w", err) - } - case "yaml": - output, err = yaml.Marshal(config) - if err != nil { - return fmt.Errorf("failed to marshal config to YAML:\n%w", err) - } - default: - return fmt.Errorf("unsupported output format %#q; use 'json' or 'yaml'", options.Format) - } - - _, err = fmt.Fprintln(os.Stdout, string(output)) - if err != nil { - return fmt.Errorf("failed to write output:\n%w", err) + return nil, fmt.Errorf("failed to filter environment:\n%w", err) } - return nil + return config, nil } diff --git a/internal/app/azldev/core/cttools/loader.go b/internal/app/azldev/core/cttools/loader.go index b8c47045..9251e1ef 100644 --- a/internal/app/azldev/core/cttools/loader.go +++ b/internal/app/azldev/core/cttools/loader.go @@ -5,24 +5,29 @@ package cttools import ( "fmt" - "os" + "log/slog" "path/filepath" + "github.com/bmatcuk/doublestar/v4" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/pelletier/go-toml/v2" ) // LoadConfig loads a distro config starting from the given top-level TOML file path. // It recursively resolves `include` directives (relative glob paths), deep-merges -// all included files, and returns the final merged raw map. -func LoadConfig(topLevelPath string) (*DistroConfig, error) { +// all included files, and returns the final merged [DistroConfig]. +func LoadConfig(fs opctx.FS, topLevelPath string) (*DistroConfig, error) { absPath, err := filepath.Abs(topLevelPath) if err != nil { return nil, fmt.Errorf("failed to resolve absolute path for %#q:\n%w", topLevelPath, err) } + slog.Debug("Loading CT distro config", "path", absPath) + visited := make(map[string]bool) - merged, err := loadAndMerge(absPath, visited) + merged, err := loadAndMerge(fs, absPath, visited) if err != nil { return nil, err } @@ -46,14 +51,16 @@ func LoadConfig(topLevelPath string) (*DistroConfig, error) { // loadAndMerge loads a single TOML file, processes its include directives, // and returns the deep-merged result as a raw map. -func loadAndMerge(absPath string, visited map[string]bool) (map[string]any, error) { +func loadAndMerge(fs opctx.FS, absPath string, visited map[string]bool) (map[string]any, error) { if visited[absPath] { return nil, fmt.Errorf("circular include detected for %#q", absPath) } visited[absPath] = true - data, err := os.ReadFile(absPath) + slog.Debug("Loading CT config file", "path", absPath) + + data, err := fileutils.ReadFile(fs, absPath) if err != nil { return nil, fmt.Errorf("failed to read %#q:\n%w", absPath, err) } @@ -80,15 +87,19 @@ func loadAndMerge(absPath string, visited map[string]bool) (map[string]any, erro for _, pattern := range includes { globPath := filepath.Join(dir, pattern) - matches, err := filepath.Glob(globPath) + slog.Debug("Resolving CT config include", "pattern", globPath, "from", absPath) + + matches, err := fileutils.Glob(fs, globPath, doublestar.WithFilesOnly()) if err != nil { return nil, fmt.Errorf("failed to glob %#q (from include in %#q):\n%w", globPath, absPath, err) } for _, match := range matches { - matchAbs, err := filepath.Abs(match) - if err != nil { - return nil, fmt.Errorf("failed to resolve absolute path for %#q:\n%w", match, err) + // fileutils.Glob may return paths relative to the FS root. + // Ensure they are absolute for consistent handling. + matchAbs := match + if !filepath.IsAbs(match) { + matchAbs = "/" + match } // Skip self-includes (e.g., when a glob like "./*.toml" matches the current file). @@ -96,9 +107,9 @@ func loadAndMerge(absPath string, visited map[string]bool) (map[string]any, erro continue } - child, err := loadAndMerge(matchAbs, visited) + child, err := loadAndMerge(fs, matchAbs, visited) if err != nil { - return nil, fmt.Errorf("error loading include %#q from %#q:\n%w", match, absPath, err) + return nil, fmt.Errorf("error loading include %#q from %#q:\n%w", matchAbs, absPath, err) } deepMergeMaps(result, child) diff --git a/internal/app/azldev/core/cttools/loader_test.go b/internal/app/azldev/core/cttools/loader_test.go index 377c5638..2a38f97a 100644 --- a/internal/app/azldev/core/cttools/loader_test.go +++ b/internal/app/azldev/core/cttools/loader_test.go @@ -4,45 +4,55 @@ package cttools_test import ( - "os" "path/filepath" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +const testConfigDir = "/testconfig" + func TestLoadConfig_SimpleFile(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + mainPath := filepath.Join(testConfigDir, "main.toml") - writeFile(t, filepath.Join(dir, "main.toml"), ` + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` [distros.testdistro] description = "Test Distro" -`) +`), fileperms.PrivateFile)) - config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) require.Contains(t, config.Distros, "testdistro") assert.Equal(t, "Test Distro", config.Distros["testdistro"].Description) } func TestLoadConfig_IncludeResolution(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) - writeFile(t, filepath.Join(dir, "main.toml"), ` + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` include = ["sub.toml"] [distros.testdistro] description = "Test Distro" -`) +`), fileperms.PrivateFile)) - writeFile(t, filepath.Join(dir, "sub.toml"), ` + subPath := filepath.Join(testConfigDir, "sub.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), subPath, []byte(` [mock-options-templates.rpm] options = ["opt1", "opt2"] -`) +`), fileperms.PrivateFile)) - config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) require.Contains(t, config.Distros, "testdistro") @@ -51,30 +61,34 @@ options = ["opt1", "opt2"] } func TestLoadConfig_NestedIncludes(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + subDir := filepath.Join(testConfigDir, "sub") - require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub"), 0o755)) + require.NoError(t, fileutils.MkdirAll(ctx.FS(), subDir)) - writeFile(t, filepath.Join(dir, "main.toml"), ` + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` include = ["sub/mid.toml"] [distros.d] description = "D" -`) +`), fileperms.PrivateFile)) - writeFile(t, filepath.Join(dir, "sub", "mid.toml"), ` + midPath := filepath.Join(subDir, "mid.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), midPath, []byte(` include = ["leaf.toml"] [mock-options-templates.rpm] options = ["a"] -`) +`), fileperms.PrivateFile)) - writeFile(t, filepath.Join(dir, "sub", "leaf.toml"), ` + leafPath := filepath.Join(subDir, "leaf.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), leafPath, []byte(` [build-root-templates.srpm] packages = ["bash"] -`) +`), fileperms.PrivateFile)) - config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) require.Contains(t, config.Distros, "d") @@ -84,25 +98,27 @@ packages = ["bash"] } func TestLoadConfig_GlobIncludes(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + tmplDir := filepath.Join(testConfigDir, "templates") - require.NoError(t, os.MkdirAll(filepath.Join(dir, "templates"), 0o755)) + require.NoError(t, fileutils.MkdirAll(ctx.FS(), tmplDir)) - writeFile(t, filepath.Join(dir, "main.toml"), ` + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` include = ["templates/*.toml"] -`) +`), fileperms.PrivateFile)) - writeFile(t, filepath.Join(dir, "templates", "mock.toml"), ` + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(tmplDir, "mock.toml"), []byte(` [mock-options-templates.rpm] options = ["opt1"] -`) +`), fileperms.PrivateFile)) - writeFile(t, filepath.Join(dir, "templates", "build.toml"), ` + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(tmplDir, "build.toml"), []byte(` [build-root-templates.srpm] packages = ["bash"] -`) +`), fileperms.PrivateFile)) - config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) require.Contains(t, config.MockOptionsTemplates, "rpm") @@ -110,21 +126,25 @@ packages = ["bash"] } func TestLoadConfig_DeepMerge_MapsMerge(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) - writeFile(t, filepath.Join(dir, "main.toml"), ` + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` include = ["extra.toml"] [distros.d1] description = "D1" -`) +`), fileperms.PrivateFile)) - writeFile(t, filepath.Join(dir, "extra.toml"), ` + extraPath := filepath.Join(testConfigDir, "extra.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), extraPath, []byte(` [distros.d2] description = "D2" -`) +`), fileperms.PrivateFile)) - config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) require.Contains(t, config.Distros, "d1") @@ -132,21 +152,25 @@ description = "D2" } func TestLoadConfig_DeepMerge_ArraysConcatenate(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() - writeFile(t, filepath.Join(dir, "main.toml"), ` + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` include = ["extra.toml"] [[distros.d.shadow-allowlists]] tag-name = "tag1" -`) +`), fileperms.PrivateFile)) - writeFile(t, filepath.Join(dir, "extra.toml"), ` + extraPath := filepath.Join(testConfigDir, "extra.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), extraPath, []byte(` [[distros.d.shadow-allowlists]] tag-name = "tag2" -`) +`), fileperms.PrivateFile)) - config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) require.Contains(t, config.Distros, "d") @@ -158,50 +182,59 @@ tag-name = "tag2" } func TestLoadConfig_CircularInclude(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + aPath := filepath.Join(testConfigDir, "a.toml") + bPath := filepath.Join(testConfigDir, "b.toml") - writeFile(t, filepath.Join(dir, "a.toml"), `include = ["b.toml"]`) - writeFile(t, filepath.Join(dir, "b.toml"), `include = ["a.toml"]`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), aPath, []byte(`include = ["b.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), bPath, []byte(`include = ["a.toml"]`), fileperms.PrivateFile)) - _, err := cttools.LoadConfig(filepath.Join(dir, "a.toml")) + _, err := cttools.LoadConfig(ctx.FS(), aPath) require.Error(t, err) assert.Contains(t, err.Error(), "circular include") } func TestLoadConfig_MissingInclude(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() - writeFile(t, filepath.Join(dir, "main.toml"), `include = ["nonexistent.toml"]`) + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + content := []byte(`include = ["nonexistent.toml"]`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, content, fileperms.PrivateFile)) // Glob returns no matches for nonexistent files, so this should succeed with empty config. - config, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) assert.Empty(t, config.Distros) } func TestLoadConfig_InvalidTOML(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) - writeFile(t, filepath.Join(dir, "main.toml"), `this is not valid toml {{{`) + mainPath := filepath.Join(testConfigDir, "main.toml") + content := []byte(`this is not valid toml {{{`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, content, fileperms.PrivateFile)) - _, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + _, err := cttools.LoadConfig(ctx.FS(), mainPath) require.Error(t, err) assert.Contains(t, err.Error(), "failed to parse TOML") } func TestLoadConfig_InvalidIncludeType(t *testing.T) { - dir := t.TempDir() + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) - writeFile(t, filepath.Join(dir, "main.toml"), `include = 42`) + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(`include = 42`), fileperms.PrivateFile)) - _, err := cttools.LoadConfig(filepath.Join(dir, "main.toml")) + _, err := cttools.LoadConfig(ctx.FS(), mainPath) require.Error(t, err) assert.Contains(t, err.Error(), "must be an array") } - -// writeFile is a test helper that writes content to a file, creating it if needed. -func writeFile(t *testing.T, path, content string) { - t.Helper() - - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) -} diff --git a/internal/app/azldev/core/cttools/resolver.go b/internal/app/azldev/core/cttools/resolver.go index e99a94fc..c6a5474a 100644 --- a/internal/app/azldev/core/cttools/resolver.go +++ b/internal/app/azldev/core/cttools/resolver.go @@ -5,15 +5,22 @@ package cttools import ( "fmt" + "sort" ) // ResolveTemplates resolves all koji target templates for every git-source-repo in every distro // version. It expands build-root references, mock-options references, and applies // environment-prefix / repo-prefix / parent-prefix to produce [ResolvedKojiTarget] entries. func ResolveTemplates(config *DistroConfig) error { - for distroName, distro := range config.Distros { - for versionName, version := range distro.Versions { - for repoName, repos := range version.GitSourceRepos { + for _, distroName := range sortedKeys(config.Distros) { + distro := config.Distros[distroName] + + for _, versionName := range sortedKeys(distro.Versions) { + version := distro.Versions[versionName] + + for _, repoName := range sortedKeys(version.GitSourceRepos) { + repos := version.GitSourceRepos[repoName] + for i := range repos { repo := &repos[i] @@ -69,17 +76,25 @@ func resolveRepoTargets(config *DistroConfig, version *Version, repo *GitSourceR var resolved []ResolvedKojiTarget - for targetName, targets := range templateSet { + for _, targetName := range sortedKeys(templateSet) { + targets := templateSet[targetName] + for _, tmpl := range targets { - rt, err := resolveOneTarget(config, version, envPrefix, repoPrefix, parentPrefix, targetName, &tmpl) + resolvedTarget, err := resolveOneTarget( + config, version, envPrefix, repoPrefix, parentPrefix, targetName, &tmpl, + ) if err != nil { return fmt.Errorf("error resolving target %#q:\n%w", targetName, err) } - resolved = append(resolved, *rt) + resolved = append(resolved, *resolvedTarget) } } + sort.Slice(resolved, func(i, j int) bool { + return resolved[i].Name < resolved[j].Name + }) + repo.ResolvedKojiTargets = resolved return nil @@ -185,3 +200,15 @@ func resolveDistTag(version *Version, fieldName string) (string, error) { return "", fmt.Errorf("unknown mock-dist-tag field %#q", fieldName) } } + +// sortedKeys returns the keys of a string-keyed map in sorted order. +func sortedKeys[V any](inputMap map[string]V) []string { + keys := make([]string, 0, len(inputMap)) + for key := range inputMap { + keys = append(keys, key) + } + + sort.Strings(keys) + + return keys +} diff --git a/scenario/clismoke_test.go b/scenario/clismoke_test.go index abeaa32c..897a8870 100644 --- a/scenario/clismoke_test.go +++ b/scenario/clismoke_test.go @@ -181,9 +181,9 @@ func TestCTToolsConfigDump(t *testing.T) { test := cmdtest.NewScenarioTest( "advanced", "ct-tools", "config-dump", - "--config", configPath, + "--ct-config", configPath, "--environment", "ct-test", - "--format", "json", + "-O", "json", ).Locally() results, err := test.Run(t) @@ -230,28 +230,3 @@ func TestCTToolsConfigDump(t *testing.T) { require.True(t, ok) assert.Contains(t, name, "td1-dev-") } - -// Tests that `azldev advanced ct-tools config-dump` outputs valid YAML. -func TestCTToolsConfigDumpYAML(t *testing.T) { - t.Parallel() - - if testing.Short() { - t.Skip("skipping long test") - } - - configPath, err := filepath.Abs("testdata/cttools/distro.toml") - require.NoError(t, err) - - test := cmdtest.NewScenarioTest( - "advanced", "ct-tools", "config-dump", - "--config", configPath, - "--environment", "ct-test", - "--format", "yaml", - ).Locally() - - results, err := test.Run(t) - require.NoError(t, err) - require.Zero(t, results.ExitCode, "stderr: %s", results.Stderr) - assert.Contains(t, results.Stdout, "testdistro") - assert.Contains(t, results.Stdout, "ct-test") -} From 73e4ba497a2fdbf54cf4d5d535be37d98ec5e1d4 Mon Sep 17 00:00:00 2001 From: Sam Meluch Date: Thu, 2 Apr 2026 01:24:42 +0000 Subject: [PATCH 3/5] fix: improve determinism and error handling in ct-tools config-dump - Error on missing non-glob includes instead of silently succeeding, matching projectconfig/loader.go behavior (glob patterns with no matches still succeed silently) - Sort available environment names in FilterEnvironment error message for stable, testable output - templateSet iteration was already sorted from prior commit --- internal/app/azldev/core/cttools/loader.go | 14 ++++++++++++++ .../app/azldev/core/cttools/loader_test.go | 18 +++++++++++++++++- internal/app/azldev/core/cttools/resolver.go | 5 +---- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/internal/app/azldev/core/cttools/loader.go b/internal/app/azldev/core/cttools/loader.go index 9251e1ef..7b860f4c 100644 --- a/internal/app/azldev/core/cttools/loader.go +++ b/internal/app/azldev/core/cttools/loader.go @@ -6,7 +6,9 @@ package cttools import ( "fmt" "log/slog" + "os" "path/filepath" + "strings" "github.com/bmatcuk/doublestar/v4" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" @@ -94,6 +96,13 @@ func loadAndMerge(fs opctx.FS, absPath string, visited map[string]bool) (map[str return nil, fmt.Errorf("failed to glob %#q (from include in %#q):\n%w", globPath, absPath, err) } + if len(matches) == 0 && !containsGlobMeta(pattern) { + return nil, fmt.Errorf( + "failed to find include file %#q referenced in %#q:\n%w", + pattern, absPath, os.ErrNotExist, + ) + } + for _, match := range matches { // fileutils.Glob may return paths relative to the FS root. // Ensure they are absolute for consistent handling. @@ -145,6 +154,11 @@ func extractIncludes(raw map[string]any, filePath string) ([]string, error) { return result, nil } +// containsGlobMeta reports whether the pattern contains glob metacharacters. +func containsGlobMeta(pattern string) bool { + return strings.ContainsAny(pattern, "*?[") +} + // deepMergeMaps merges src into dst recursively. For map values, sub-maps are merged recursively. // For slice values, slices are concatenated. For all other types, src overwrites dst. func deepMergeMaps(dst, src map[string]any) { diff --git a/internal/app/azldev/core/cttools/loader_test.go b/internal/app/azldev/core/cttools/loader_test.go index 2a38f97a..0977d94e 100644 --- a/internal/app/azldev/core/cttools/loader_test.go +++ b/internal/app/azldev/core/cttools/loader_test.go @@ -4,6 +4,7 @@ package cttools_test import ( + "os" "path/filepath" "testing" @@ -206,7 +207,22 @@ func TestLoadConfig_MissingInclude(t *testing.T) { content := []byte(`include = ["nonexistent.toml"]`) require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, content, fileperms.PrivateFile)) - // Glob returns no matches for nonexistent files, so this should succeed with empty config. + // Non-glob include that doesn't exist should produce an error. + _, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.Error(t, err) + assert.ErrorIs(t, err, os.ErrNotExist) +} + +func TestLoadConfig_MissingGlobInclude(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + content := []byte(`include = ["nonexistent/*.toml"]`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, content, fileperms.PrivateFile)) + + // Glob pattern with no matches should silently succeed. config, err := cttools.LoadConfig(ctx.FS(), mainPath) require.NoError(t, err) assert.Empty(t, config.Distros) diff --git a/internal/app/azldev/core/cttools/resolver.go b/internal/app/azldev/core/cttools/resolver.go index c6a5474a..a3393a8c 100644 --- a/internal/app/azldev/core/cttools/resolver.go +++ b/internal/app/azldev/core/cttools/resolver.go @@ -46,10 +46,7 @@ func ResolveTemplates(config *DistroConfig) error { func FilterEnvironment(config *DistroConfig, envName string) error { env, ok := config.Environments[envName] if !ok { - available := make([]string, 0, len(config.Environments)) - for k := range config.Environments { - available = append(available, k) - } + available := sortedKeys(config.Environments) return fmt.Errorf("environment %#q not found; available: %v", envName, available) } From 3ec2824bb2ba58c0f5d38be38770d7b9d3cf3db6 Mon Sep 17 00:00:00 2001 From: Sam Meluch Date: Thu, 2 Apr 2026 17:54:28 +0000 Subject: [PATCH 4/5] fix: use repo-relative path for scenario test data in ct-tools test The CI runner's working directory is the repo root, not scenario/, so filepath.Abs(testdata/...) resolved incorrectly. Use scenario/testdata/... to match the convention in other scenario tests. --- .../cli/azldev_advanced_ct-tools_config-dump.md | 10 ++++------ scenario/clismoke_test.go | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md b/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md index 063fd38a..e1d11b6c 100644 --- a/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md +++ b/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md @@ -19,18 +19,16 @@ azldev advanced ct-tools config-dump [flags] ``` # Dump config for ct-dev as JSON - azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-dev - - # Dump config for ct-prod as YAML - azldev advanced ct-tools config-dump --config /path/to/azurelinux.toml --environment ct-prod --format yaml + azldev advanced ct-tools config-dump \ + --ct-config /path/to/azurelinux.toml \ + --environment ct-dev -O json ``` ### Options ``` - --config string Path to the top-level TOML configuration file + --ct-config string Path to the top-level CT distro TOML configuration file --environment string Control Tower environment name (e.g. ct-dev, ct-staging, ct-prod) - --format string Output format: json or yaml (default "json") -h, --help help for config-dump ``` diff --git a/scenario/clismoke_test.go b/scenario/clismoke_test.go index 897a8870..9cf390a6 100644 --- a/scenario/clismoke_test.go +++ b/scenario/clismoke_test.go @@ -176,7 +176,7 @@ func TestCTToolsConfigDump(t *testing.T) { } // Use the self-contained test config under scenario/testdata/cttools/. - configPath, err := filepath.Abs("testdata/cttools/distro.toml") + configPath, err := filepath.Abs("scenario/testdata/cttools/distro.toml") require.NoError(t, err) test := cmdtest.NewScenarioTest( From f3d59de8868d2e13020e04fdc86799e8aab49a61 Mon Sep 17 00:00:00 2001 From: Sam Meluch Date: Thu, 2 Apr 2026 21:39:42 +0000 Subject: [PATCH 5/5] fix: replace visited set with recursion stack for cycle detection The visited map was never cleared on recursion unwind, so diamond includes (same file included from two branches) were incorrectly flagged as circular. Replace with an inProgress stack (defer delete on exit) for true cycle detection and a separate loaded set to skip already-processed files. Extract resolveIncludes helper to stay within funlen limit. Add TestLoadConfig_DiamondInclude to cover the diamond pattern. --- internal/app/azldev/core/cttools/loader.go | 52 ++++++++++++++----- .../app/azldev/core/cttools/loader_test.go | 30 +++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/internal/app/azldev/core/cttools/loader.go b/internal/app/azldev/core/cttools/loader.go index 7b860f4c..c31774f9 100644 --- a/internal/app/azldev/core/cttools/loader.go +++ b/internal/app/azldev/core/cttools/loader.go @@ -27,9 +27,10 @@ func LoadConfig(fs opctx.FS, topLevelPath string) (*DistroConfig, error) { slog.Debug("Loading CT distro config", "path", absPath) - visited := make(map[string]bool) + inProgress := make(map[string]bool) + loaded := make(map[string]bool) - merged, err := loadAndMerge(fs, absPath, visited) + merged, err := loadAndMerge(fs, absPath, inProgress, loaded) if err != nil { return nil, err } @@ -52,13 +53,26 @@ func LoadConfig(fs opctx.FS, topLevelPath string) (*DistroConfig, error) { } // loadAndMerge loads a single TOML file, processes its include directives, -// and returns the deep-merged result as a raw map. -func loadAndMerge(fs opctx.FS, absPath string, visited map[string]bool) (map[string]any, error) { - if visited[absPath] { +// and returns the deep-merged result as a raw map. inProgress tracks the current +// recursion stack to detect cycles. loaded tracks files that have already been +// fully processed so that diamond includes (e.g. common.toml included from two +// branches) are only merged once. +func loadAndMerge( + fs opctx.FS, absPath string, inProgress map[string]bool, loaded map[string]bool, +) (map[string]any, error) { + if inProgress[absPath] { return nil, fmt.Errorf("circular include detected for %#q", absPath) } - visited[absPath] = true + // Skip files already loaded from another branch to avoid duplicate merging. + if loaded[absPath] { + slog.Debug("Skipping already-loaded CT config file", "path", absPath) + + return make(map[string]any), nil + } + + inProgress[absPath] = true + defer delete(inProgress, absPath) slog.Debug("Loading CT config file", "path", absPath) @@ -83,7 +97,21 @@ func loadAndMerge(fs opctx.FS, absPath string, visited map[string]bool) (map[str deepMergeMaps(result, raw) delete(result, "include") - // Load each included file and merge into result. + if err := resolveIncludes(fs, absPath, includes, inProgress, loaded, result); err != nil { + return nil, err + } + + loaded[absPath] = true + + return result, nil +} + +// resolveIncludes processes include directives for a single config file, loading +// and deep-merging each included file's content into result. +func resolveIncludes( + fs opctx.FS, absPath string, includes []string, + inProgress map[string]bool, loaded map[string]bool, result map[string]any, +) error { dir := filepath.Dir(absPath) for _, pattern := range includes { @@ -93,11 +121,11 @@ func loadAndMerge(fs opctx.FS, absPath string, visited map[string]bool) (map[str matches, err := fileutils.Glob(fs, globPath, doublestar.WithFilesOnly()) if err != nil { - return nil, fmt.Errorf("failed to glob %#q (from include in %#q):\n%w", globPath, absPath, err) + return fmt.Errorf("failed to glob %#q (from include in %#q):\n%w", globPath, absPath, err) } if len(matches) == 0 && !containsGlobMeta(pattern) { - return nil, fmt.Errorf( + return fmt.Errorf( "failed to find include file %#q referenced in %#q:\n%w", pattern, absPath, os.ErrNotExist, ) @@ -116,16 +144,16 @@ func loadAndMerge(fs opctx.FS, absPath string, visited map[string]bool) (map[str continue } - child, err := loadAndMerge(fs, matchAbs, visited) + child, err := loadAndMerge(fs, matchAbs, inProgress, loaded) if err != nil { - return nil, fmt.Errorf("error loading include %#q from %#q:\n%w", matchAbs, absPath, err) + return fmt.Errorf("error loading include %#q from %#q:\n%w", matchAbs, absPath, err) } deepMergeMaps(result, child) } } - return result, nil + return nil } // extractIncludes reads the "include" key from a raw TOML map and returns it as a string slice. diff --git a/internal/app/azldev/core/cttools/loader_test.go b/internal/app/azldev/core/cttools/loader_test.go index 0977d94e..38ec935b 100644 --- a/internal/app/azldev/core/cttools/loader_test.go +++ b/internal/app/azldev/core/cttools/loader_test.go @@ -198,6 +198,36 @@ func TestLoadConfig_CircularInclude(t *testing.T) { assert.Contains(t, err.Error(), "circular include") } +func TestLoadConfig_DiamondInclude(t *testing.T) { + // A diamond include pattern: root includes both a.toml and b.toml, + // and both a.toml and b.toml include common.toml. This must not be + // treated as a circular include. + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + rootPath := filepath.Join(testConfigDir, "root.toml") + aPath := filepath.Join(testConfigDir, "a.toml") + bPath := filepath.Join(testConfigDir, "b.toml") + commonPath := filepath.Join(testConfigDir, "common.toml") + + require.NoError(t, fileutils.WriteFile(ctx.FS(), rootPath, + []byte(`include = ["a.toml", "b.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), aPath, + []byte(`include = ["common.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), bPath, + []byte(`include = ["common.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), commonPath, []byte(` +[mock-options-templates.shared] +options = ["shared-opt"] +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), rootPath) + require.NoError(t, err) + require.Contains(t, config.MockOptionsTemplates, "shared") + assert.Equal(t, []string{"shared-opt"}, config.MockOptionsTemplates["shared"].Options) +} + func TestLoadConfig_MissingInclude(t *testing.T) { ctx := testctx.NewCtx()