Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions docs/user/reference/config/components.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ A component definition tells azldev where to find the spec file, how to customiz
| Release config | `release` | [ReleaseConfig](#release-configuration) | No | Controls how the Release tag is managed during rendering |
| Overlays | `overlays` | array of [Overlay](overlays.md) | No | Modifications to apply to the spec and/or source files |
| Build config | `build` | [BuildConfig](#build-configuration) | No | Build-time options (macros, conditionals, check config) |
| Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior |
| Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component |
| Default package config | `default-package-config` | [PackageConfig](package-groups.md#package-config) | No | Default configuration applied to all binary packages produced by this component; overrides project defaults and package-group defaults |
| Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order |
Expand Down Expand Up @@ -113,6 +114,27 @@ Most components use auto mode (the default) and need no release configuration. S
calculation = "manual"
```

## Render Configuration

The `[components.<name>.render]` section controls rendering behavior for a component.

| Field | TOML Key | Type | Required | Description |
|-------|----------|------|----------|-------------|
| Skip file filter | `skip-file-filter` | boolean | No | Disable post-render file filtering (defaults to `false`) |

### Skip File Filter

During rendering, azldev uses `spectool` to determine which files are referenced by `Source` and `Patch` tags in the spec, then removes unreferenced files from the rendered output. Some specs use dynamic macros (e.g., `%{fontpkgname1}`) that `spectool` cannot expand, causing it to report incorrect filenames. This results in referenced files being incorrectly removed.

Set `skip-file-filter = true` to preserve all files from the dist-git checkout:

```toml
[components.dejavu-fonts.render]
skip-file-filter = true
```

> **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most components, the default filtering behavior is correct and keeps the rendered output clean.

## Build Configuration

The `[components.<name>.build]` section controls build-time options for a component.
Expand Down
26 changes: 25 additions & 1 deletion internal/app/azldev/cmds/component/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,16 @@ func finishComponentRender(
}

// Filter files using spectool result from batch mock.
if filterErr := removeUnreferencedFiles(
// Skip filtering when:
// 1. The component config explicitly opts out via 'skip-file-filter'.
// 2. spectool output contains unexpanded RPM macros (%{...}), indicating
// that the reported filenames don't match the real files on disk.
if prep.comp.GetConfig().Render.SkipFileFilter {
slog.Info("Skipping file filter ('skip-file-filter' is set)", "component", componentName)
} else if macro := findUnexpandedMacro(mockResult.SpecFiles); macro != "" {
slog.Info("Skipping file filter (spectool output contains unexpanded macros)",
"component", componentName, "example", macro)
} else if filterErr := removeUnreferencedFiles(
env.FS(), componentDir, specPath, mockResult.SpecFiles, componentName,
); filterErr != nil {
return fmt.Errorf("filtering unreferenced files for %#q:\n%w", componentName, filterErr)
Expand Down Expand Up @@ -700,6 +709,21 @@ func copyRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, all
return nil
}

// findUnexpandedMacro returns the first filename from specFiles that contains
// an unexpanded RPM macro (i.e., a literal "%{...}" sequence), or "" if all
// macros were resolved. When spectool cannot resolve a macro, it emits the raw
// macro text as part of the filename (e.g., "57-%{fontpkgname1}.xml"), which
// won't match any real file on disk.
func findUnexpandedMacro(specFiles []string) string {
for _, f := range specFiles {
if strings.Contains(f, "%{") {
return f
}
}

return ""
}

// removeUnreferencedFiles removes files from the directory that aren't in the keep-list.
// The keep-list is built from the spec file, the "sources" directory, and all
// source/patch filenames provided. For paths with subdirectories (e.g., "patches/fix.patch"),
Expand Down
97 changes: 97 additions & 0 deletions internal/app/azldev/cmds/component/render_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -264,3 +264,100 @@ func TestRemoveUnreferencedFiles(t *testing.T) {
assert.Len(t, entries, 2, "both files should remain")
})
}

func TestSkipFileFilterPreservesAllFiles(t *testing.T) {
// Verifies that when SkipFileFilter is true, unreferenced files are NOT removed.
// This mirrors the finishComponentRender logic: when the flag is set,
// removeUnreferencedFiles is not called, so all files survive.
testFS := afero.NewMemMapFs()

require.NoError(t, fileutils.MkdirAll(testFS, "/render"))
require.NoError(t, fileutils.WriteFile(testFS, "/render/pkg.spec", []byte("spec"), fileperms.PublicFile))
require.NoError(t, fileutils.WriteFile(testFS, "/render/sources", []byte("hash"), fileperms.PublicFile))
require.NoError(t, fileutils.WriteFile(testFS, "/render/57-pkg-fonts.xml", []byte("fontconfig"), fileperms.PublicFile))
require.NoError(t, fileutils.WriteFile(
testFS, "/render/58-pkg-lgc-fonts.xml", []byte("fontconfig"), fileperms.PublicFile))

// spectool would report unexpanded macros like "57-%{fontpkgname1}.xml"
// which don't match any file on disk. Without skip-file-filter, the
// filter would delete the real XML files.
specFiles := []string{"57-%{fontpkgname1}.xml", "58-%{fontpkgname4}.xml"}

// Simulate skip-file-filter=false: XML files get removed.
err := removeUnreferencedFiles(testFS, "/render", "/render/pkg.spec", specFiles, "pkg")
require.NoError(t, err)

for _, name := range []string{"57-pkg-fonts.xml", "58-pkg-lgc-fonts.xml"} {
exists, existsErr := fileutils.Exists(testFS, filepath.Join("/render", name))
require.NoError(t, existsErr)
assert.False(t, exists, "%s should be removed when skip-file-filter is false", name)
}

// Simulate skip-file-filter=true: removeUnreferencedFiles is never called,
// so all files are preserved. Reset the filesystem and verify.
testFS = afero.NewMemMapFs()

require.NoError(t, fileutils.MkdirAll(testFS, "/render"))
require.NoError(t, fileutils.WriteFile(testFS, "/render/pkg.spec", []byte("spec"), fileperms.PublicFile))
require.NoError(t, fileutils.WriteFile(testFS, "/render/sources", []byte("hash"), fileperms.PublicFile))
require.NoError(t, fileutils.WriteFile(testFS, "/render/57-pkg-fonts.xml", []byte("fontconfig"), fileperms.PublicFile))
require.NoError(t, fileutils.WriteFile(
testFS, "/render/58-pkg-lgc-fonts.xml", []byte("fontconfig"), fileperms.PublicFile))

// With skip-file-filter=true, removeUnreferencedFiles is NOT called.
// All files should remain.
for _, name := range []string{"pkg.spec", "sources", "57-pkg-fonts.xml", "58-pkg-lgc-fonts.xml"} {
exists, existsErr := fileutils.Exists(testFS, filepath.Join("/render", name))
require.NoError(t, existsErr)
assert.True(t, exists, "%s should be preserved when skip-file-filter is true", name)
}
}

func TestFindUnexpandedMacro(t *testing.T) {
t.Parallel()

tests := []struct {
name string
specFiles []string
want string
}{
{
name: "no macros",
specFiles: []string{"curl-8.0.tar.xz", "fix.patch"},
want: "",
},
{
name: "one unexpanded macro",
specFiles: []string{"curl-8.0.tar.xz", "57-%{fontpkgname1}.xml"},
want: "57-%{fontpkgname1}.xml",
},
{
name: "returns first match",
specFiles: []string{
"good.tar.gz",
"57-%{fontpkgname1}.xml",
"58-%{fontpkgname4}.xml",
},
want: "57-%{fontpkgname1}.xml",
},
{
name: "empty input",
specFiles: nil,
want: "",
},
{
name: "rust crates_source macro",
specFiles: []string{"%{crates_source}"},
want: "%{crates_source}",
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

result := findUnexpandedMacro(tc.specFiles)
assert.Equal(t, tc.want, result)
})
}
}
4 changes: 4 additions & 0 deletions internal/projectconfig/component.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ type ComponentConfig struct {
// Configuration for building the component.
Build ComponentBuildConfig `toml:"build,omitempty" json:"build,omitempty" table:"-" jsonschema:"title=Build configuration,description=Configuration for building the component"`

// Configuration for rendering the component.
Render ComponentRenderConfig `toml:"render,omitempty" json:"render,omitempty" table:"-" jsonschema:"title=Render configuration,description=Configuration for rendering the component"`

Comment thread
dmcilvaney marked this conversation as resolved.
// Source file references for this component.
SourceFiles []SourceFileReference `toml:"source-files,omitempty" json:"sourceFiles,omitempty" table:"-" jsonschema:"title=Source files,description=Source files to download for this component"`

Expand Down Expand Up @@ -237,6 +240,7 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi
Release: c.Release,
Spec: deep.MustCopy(c.Spec),
Build: deep.MustCopy(c.Build),
Render: c.Render,
SourceFiles: deep.MustCopy(c.SourceFiles),
DefaultPackageConfig: deep.MustCopy(c.DefaultPackageConfig),
Packages: deep.MustCopy(c.Packages),
Expand Down
1 change: 1 addition & 0 deletions internal/projectconfig/fingerprint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) {
reflect.TypeFor[projectconfig.DistroReference](),
reflect.TypeFor[projectconfig.SourceFileReference](),
reflect.TypeFor[projectconfig.ReleaseConfig](),
reflect.TypeFor[projectconfig.ComponentRenderConfig](),
}

// Maps "StructName.FieldName" for every field that should carry a
Expand Down
15 changes: 15 additions & 0 deletions internal/projectconfig/render.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

package projectconfig

// ComponentRenderConfig encapsulates configuration for rendering a component.
type ComponentRenderConfig struct {
// SkipFileFilter, when true, disables the post-render file filter for this
// component. Normally, rendered output is filtered to only include files
// referenced by Source/Patch tags in the spec (as reported by spectool).
// Some specs use macros that spectool cannot expand, causing referenced
// files to be incorrectly removed. Setting this to true preserves all
// files from the dist-git checkout.
SkipFileFilter bool `toml:"skip-file-filter,omitempty" json:"skipFileFilter,omitempty" jsonschema:"title=Skip file filter,description=Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags"`
}
Comment thread
dmcilvaney marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@
"title": "Build configuration",
"description": "Configuration for building the component"
},
"render": {
"$ref": "#/$defs/ComponentRenderConfig",
"title": "Render configuration",
"description": "Configuration for rendering the component"
},
"source-files": {
"items": {
"$ref": "#/$defs/SourceFileReference"
Expand Down Expand Up @@ -280,6 +285,17 @@
"type"
]
},
"ComponentRenderConfig": {
"properties": {
"skip-file-filter": {
"type": "boolean",
"title": "Skip file filter",
"description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags"
}
},
"additionalProperties": false,
"type": "object"
},
"ConfigFile": {
"properties": {
"$schema": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@
"title": "Build configuration",
"description": "Configuration for building the component"
},
"render": {
"$ref": "#/$defs/ComponentRenderConfig",
"title": "Render configuration",
"description": "Configuration for rendering the component"
},
"source-files": {
"items": {
"$ref": "#/$defs/SourceFileReference"
Expand Down Expand Up @@ -280,6 +285,17 @@
"type"
]
},
"ComponentRenderConfig": {
"properties": {
"skip-file-filter": {
"type": "boolean",
"title": "Skip file filter",
"description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags"
}
},
"additionalProperties": false,
"type": "object"
},
"ConfigFile": {
"properties": {
"$schema": {
Expand Down
16 changes: 16 additions & 0 deletions schemas/azldev.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@
"title": "Build configuration",
"description": "Configuration for building the component"
},
"render": {
"$ref": "#/$defs/ComponentRenderConfig",
"title": "Render configuration",
"description": "Configuration for rendering the component"
},
"source-files": {
"items": {
"$ref": "#/$defs/SourceFileReference"
Expand Down Expand Up @@ -280,6 +285,17 @@
"type"
]
},
"ComponentRenderConfig": {
"properties": {
"skip-file-filter": {
"type": "boolean",
"title": "Skip file filter",
"description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags"
}
},
"additionalProperties": false,
"type": "object"
},
"ConfigFile": {
"properties": {
"$schema": {
Expand Down
Loading