diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 78b39579..d61315b4 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -12,6 +12,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Release config | `release` | [ReleaseConfig](#release-configuration) | No | Controls how the Release tag is managed during rendering | | Overlays | `overlays` | array of [Overlay](overlays.md) | No | Modifications to apply to the spec and/or source files | | Build config | `build` | [BuildConfig](#build-configuration) | No | Build-time options (macros, conditionals, check config) | +| Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | | Default package config | `default-package-config` | [PackageConfig](package-groups.md#package-config) | No | Default configuration applied to all binary packages produced by this component; overrides project defaults and package-group defaults | | Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order | @@ -113,6 +114,27 @@ Most components use auto mode (the default) and need no release configuration. S calculation = "manual" ``` +## Render Configuration + +The `[components..render]` section controls rendering behavior for a component. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Skip file filter | `skip-file-filter` | boolean | No | Disable post-render file filtering (defaults to `false`) | + +### Skip File Filter + +During rendering, azldev uses `spectool` to determine which files are referenced by `Source` and `Patch` tags in the spec, then removes unreferenced files from the rendered output. Some specs use dynamic macros (e.g., `%{fontpkgname1}`) that `spectool` cannot expand, causing it to report incorrect filenames. This results in referenced files being incorrectly removed. + +Set `skip-file-filter = true` to preserve all files from the dist-git checkout: + +```toml +[components.dejavu-fonts.render] +skip-file-filter = true +``` + +> **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most components, the default filtering behavior is correct and keeps the rendered output clean. + ## Build Configuration The `[components..build]` section controls build-time options for a component. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 20126af2..b10b1a3c 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -636,7 +636,16 @@ func finishComponentRender( } // Filter files using spectool result from batch mock. - if filterErr := removeUnreferencedFiles( + // Skip filtering when: + // 1. The component config explicitly opts out via 'skip-file-filter'. + // 2. spectool output contains unexpanded RPM macros (%{...}), indicating + // that the reported filenames don't match the real files on disk. + if prep.comp.GetConfig().Render.SkipFileFilter { + slog.Info("Skipping file filter ('skip-file-filter' is set)", "component", componentName) + } else if macro := findUnexpandedMacro(mockResult.SpecFiles); macro != "" { + slog.Info("Skipping file filter (spectool output contains unexpanded macros)", + "component", componentName, "example", macro) + } else if filterErr := removeUnreferencedFiles( env.FS(), componentDir, specPath, mockResult.SpecFiles, componentName, ); filterErr != nil { return fmt.Errorf("filtering unreferenced files for %#q:\n%w", componentName, filterErr) @@ -700,6 +709,21 @@ func copyRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, all return nil } +// findUnexpandedMacro returns the first filename from specFiles that contains +// an unexpanded RPM macro (i.e., a literal "%{...}" sequence), or "" if all +// macros were resolved. When spectool cannot resolve a macro, it emits the raw +// macro text as part of the filename (e.g., "57-%{fontpkgname1}.xml"), which +// won't match any real file on disk. +func findUnexpandedMacro(specFiles []string) string { + for _, f := range specFiles { + if strings.Contains(f, "%{") { + return f + } + } + + return "" +} + // removeUnreferencedFiles removes files from the directory that aren't in the keep-list. // The keep-list is built from the spec file, the "sources" directory, and all // source/patch filenames provided. For paths with subdirectories (e.g., "patches/fix.patch"), diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 85f7d420..185e0a5c 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -264,3 +264,100 @@ func TestRemoveUnreferencedFiles(t *testing.T) { assert.Len(t, entries, 2, "both files should remain") }) } + +func TestSkipFileFilterPreservesAllFiles(t *testing.T) { + // Verifies that when SkipFileFilter is true, unreferenced files are NOT removed. + // This mirrors the finishComponentRender logic: when the flag is set, + // removeUnreferencedFiles is not called, so all files survive. + testFS := afero.NewMemMapFs() + + require.NoError(t, fileutils.MkdirAll(testFS, "/render")) + require.NoError(t, fileutils.WriteFile(testFS, "/render/pkg.spec", []byte("spec"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(testFS, "/render/sources", []byte("hash"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(testFS, "/render/57-pkg-fonts.xml", []byte("fontconfig"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile( + testFS, "/render/58-pkg-lgc-fonts.xml", []byte("fontconfig"), fileperms.PublicFile)) + + // spectool would report unexpanded macros like "57-%{fontpkgname1}.xml" + // which don't match any file on disk. Without skip-file-filter, the + // filter would delete the real XML files. + specFiles := []string{"57-%{fontpkgname1}.xml", "58-%{fontpkgname4}.xml"} + + // Simulate skip-file-filter=false: XML files get removed. + err := removeUnreferencedFiles(testFS, "/render", "/render/pkg.spec", specFiles, "pkg") + require.NoError(t, err) + + for _, name := range []string{"57-pkg-fonts.xml", "58-pkg-lgc-fonts.xml"} { + exists, existsErr := fileutils.Exists(testFS, filepath.Join("/render", name)) + require.NoError(t, existsErr) + assert.False(t, exists, "%s should be removed when skip-file-filter is false", name) + } + + // Simulate skip-file-filter=true: removeUnreferencedFiles is never called, + // so all files are preserved. Reset the filesystem and verify. + testFS = afero.NewMemMapFs() + + require.NoError(t, fileutils.MkdirAll(testFS, "/render")) + require.NoError(t, fileutils.WriteFile(testFS, "/render/pkg.spec", []byte("spec"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(testFS, "/render/sources", []byte("hash"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(testFS, "/render/57-pkg-fonts.xml", []byte("fontconfig"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile( + testFS, "/render/58-pkg-lgc-fonts.xml", []byte("fontconfig"), fileperms.PublicFile)) + + // With skip-file-filter=true, removeUnreferencedFiles is NOT called. + // All files should remain. + for _, name := range []string{"pkg.spec", "sources", "57-pkg-fonts.xml", "58-pkg-lgc-fonts.xml"} { + exists, existsErr := fileutils.Exists(testFS, filepath.Join("/render", name)) + require.NoError(t, existsErr) + assert.True(t, exists, "%s should be preserved when skip-file-filter is true", name) + } +} + +func TestFindUnexpandedMacro(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + specFiles []string + want string + }{ + { + name: "no macros", + specFiles: []string{"curl-8.0.tar.xz", "fix.patch"}, + want: "", + }, + { + name: "one unexpanded macro", + specFiles: []string{"curl-8.0.tar.xz", "57-%{fontpkgname1}.xml"}, + want: "57-%{fontpkgname1}.xml", + }, + { + name: "returns first match", + specFiles: []string{ + "good.tar.gz", + "57-%{fontpkgname1}.xml", + "58-%{fontpkgname4}.xml", + }, + want: "57-%{fontpkgname1}.xml", + }, + { + name: "empty input", + specFiles: nil, + want: "", + }, + { + name: "rust crates_source macro", + specFiles: []string{"%{crates_source}"}, + want: "%{crates_source}", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + result := findUnexpandedMacro(tc.specFiles) + assert.Equal(t, tc.want, result) + }) + } +} diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 8e8a9137..4d59035c 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -156,6 +156,9 @@ type ComponentConfig struct { // Configuration for building the component. Build ComponentBuildConfig `toml:"build,omitempty" json:"build,omitempty" table:"-" jsonschema:"title=Build configuration,description=Configuration for building the component"` + // Configuration for rendering the component. + Render ComponentRenderConfig `toml:"render,omitempty" json:"render,omitempty" table:"-" jsonschema:"title=Render configuration,description=Configuration for rendering the component"` + // Source file references for this component. SourceFiles []SourceFileReference `toml:"source-files,omitempty" json:"sourceFiles,omitempty" table:"-" jsonschema:"title=Source files,description=Source files to download for this component"` @@ -237,6 +240,7 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi Release: c.Release, Spec: deep.MustCopy(c.Spec), Build: deep.MustCopy(c.Build), + Render: c.Render, SourceFiles: deep.MustCopy(c.SourceFiles), DefaultPackageConfig: deep.MustCopy(c.DefaultPackageConfig), Packages: deep.MustCopy(c.Packages), diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 2cdd8a9f..32b32bdd 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -33,6 +33,7 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { reflect.TypeFor[projectconfig.DistroReference](), reflect.TypeFor[projectconfig.SourceFileReference](), reflect.TypeFor[projectconfig.ReleaseConfig](), + reflect.TypeFor[projectconfig.ComponentRenderConfig](), } // Maps "StructName.FieldName" for every field that should carry a diff --git a/internal/projectconfig/render.go b/internal/projectconfig/render.go new file mode 100644 index 00000000..c623f8af --- /dev/null +++ b/internal/projectconfig/render.go @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +// ComponentRenderConfig encapsulates configuration for rendering a component. +type ComponentRenderConfig struct { + // SkipFileFilter, when true, disables the post-render file filter for this + // component. Normally, rendered output is filtered to only include files + // referenced by Source/Patch tags in the spec (as reported by spectool). + // Some specs use macros that spectool cannot expand, causing referenced + // files to be incorrectly removed. Setting this to true preserves all + // files from the dist-git checkout. + SkipFileFilter bool `toml:"skip-file-filter,omitempty" json:"skipFileFilter,omitempty" jsonschema:"title=Skip file filter,description=Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags"` +} diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index d90a8344..f9dd59a2 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -124,6 +124,11 @@ "title": "Build configuration", "description": "Configuration for building the component" }, + "render": { + "$ref": "#/$defs/ComponentRenderConfig", + "title": "Render configuration", + "description": "Configuration for rendering the component" + }, "source-files": { "items": { "$ref": "#/$defs/SourceFileReference" @@ -280,6 +285,17 @@ "type" ] }, + "ComponentRenderConfig": { + "properties": { + "skip-file-filter": { + "type": "boolean", + "title": "Skip file filter", + "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index d90a8344..f9dd59a2 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -124,6 +124,11 @@ "title": "Build configuration", "description": "Configuration for building the component" }, + "render": { + "$ref": "#/$defs/ComponentRenderConfig", + "title": "Render configuration", + "description": "Configuration for rendering the component" + }, "source-files": { "items": { "$ref": "#/$defs/SourceFileReference" @@ -280,6 +285,17 @@ "type" ] }, + "ComponentRenderConfig": { + "properties": { + "skip-file-filter": { + "type": "boolean", + "title": "Skip file filter", + "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index d90a8344..f9dd59a2 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -124,6 +124,11 @@ "title": "Build configuration", "description": "Configuration for building the component" }, + "render": { + "$ref": "#/$defs/ComponentRenderConfig", + "title": "Render configuration", + "description": "Configuration for rendering the component" + }, "source-files": { "items": { "$ref": "#/$defs/SourceFileReference" @@ -280,6 +285,17 @@ "type" ] }, + "ComponentRenderConfig": { + "properties": { + "skip-file-filter": { + "type": "boolean", + "title": "Skip file filter", + "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": {