From 2dd31793cc33bb5ffcae47e8698d3e9be00bc589 Mon Sep 17 00:00:00 2001 From: reuben olinsky Date: Mon, 20 Apr 2026 15:17:11 +0000 Subject: [PATCH] feat: add pytest test suite support Add config-driven pytest test suites with typed TestSuiteConfig. Test suites are defined in [test-suites] with type = "pytest" and a [pytest] subtable specifying working-dir, test-paths (glob-expanded), and extra-args (with {image-path} placeholder substitution). Rewrite the 'image test' command to resolve test suites from project config, dispatch to the pytest runner, and support --test-suite filtering, --junit-xml output, and automatic image artifact resolution. Local LISA-specific test runner code is temporarily replaced; we will bring it back with the new TOML-driven configuration for local test execution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/user/reference/cli/azldev_image_test.md | 52 ++- internal/app/azldev/cmds/image/boot.go | 70 ++-- internal/app/azldev/cmds/image/boot_test.go | 16 +- .../app/azldev/cmds/image/pytestrunner.go | 356 ++++++++++++++++++ .../azldev/cmds/image/pytestrunner_test.go | 226 +++++++++++ internal/app/azldev/cmds/image/test.go | 344 ++++++++--------- internal/app/azldev/cmds/image/test_test.go | 59 +-- internal/projectconfig/configfile.go | 9 + internal/projectconfig/loader.go | 12 +- internal/projectconfig/loader_test.go | 109 +++++- internal/projectconfig/project.go | 8 +- internal/projectconfig/testsuite.go | 168 ++++++++- internal/projectconfig/testsuite_test.go | 181 +++++++++ ...ainer_config_generate-schema_stdout_1.snap | 55 ++- ...shots_config_generate-schema_stdout_1.snap | 55 ++- schemas/azldev.schema.json | 55 ++- 16 files changed, 1483 insertions(+), 292 deletions(-) create mode 100644 internal/app/azldev/cmds/image/pytestrunner.go create mode 100644 internal/app/azldev/cmds/image/pytestrunner_test.go diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index 209e4dd6..89c29c12 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -6,39 +6,55 @@ Run tests against an Azure Linux image ### Synopsis -Run tests against an Azure Linux image using a supported test runner. +Run tests against an Azure Linux image using test suites defined in the +project configuration. -Currently only the LISA test runner is supported. The image must be in qcow2, -vhd, or vhdfixed format. If the image is in vhd/vhdfixed format it is -automatically converted to qcow2 before running the tests. +Test suites are defined in the [test-suites] section of azldev.toml and referenced +by images via the [images.NAME.tests] subtable. Each test suite specifies a type +and framework-specific configuration in a matching subtable. -Requirements: - - lisa (Installation instructions: https://github.com/microsoft/lisa/blob/main/INSTALL.md) - - runbook file (YAML format defining the tests to run: https://github.com/microsoft/lisa/blob/main/docs/Runbooks.md) - - qemu-img (for vhd/vhdfixed to qcow2 conversion, if needed) +By default, all test suites associated with the named image are run. Use +--test-suite to select specific suites (may be repeated). + +The image artifact can be specified explicitly with --image-path, or resolved +automatically from the image name in the output directory. + +For pytest tests, azldev creates a Python virtual environment, installs +dependencies from pyproject.toml in the working directory, and runs pytest +with the configured test paths and extra arguments. Use {image-path} in +extra-args to insert the image path. Glob patterns (including **) in +test-paths are expanded automatically. ``` -azldev image test [flags] +azldev image test IMAGE_NAME [flags] ``` ### Examples ``` - # Run LISA tests against a qcow2 image - azldev image test --image-path ./out/image.qcow2 --test-runner lisa --runbook-path ./runbooks/smoke.yml + # Run all test suites for an image (artifact auto-resolved from output dir) + azldev image test vm-base + + # Run all test suites with an explicit image path + azldev image test vm-base --image-path ./out/images/vm-base/image.raw + + # Run a specific test suite + azldev image test vm-base --test-suite common-vm-checks + + # Run multiple specific test suites + azldev image test vm-base --test-suite common-vm-checks --test-suite vm-base-checks - # Run LISA tests against a vhd image (auto-converted to qcow2) - azldev image test --image-path ./out/image.vhd --test-runner lisa --runbook-path ./runbooks/smoke.yml + # Generate JUnit XML output + azldev image test vm-base --junit-xml results.xml ``` ### Options ``` - -k, --admin-private-key-path string Path to the admin SSH private key file passed to LISA - -h, --help help for test - -i, --image-path string Path to the disk image file to test - -r, --runbook-path string Path to the test runbook file - --test-runner string Test runner to use (currently only 'lisa' is supported) + -h, --help help for test + -i, --image-path string Path to the disk image file (resolved from image name if not specified) + --junit-xml string Path for writing JUnit XML output + --test-suite strings Name of a test suite to run (may be repeated; defaults to all suites for the image) ``` ### Options inherited from parent commands diff --git a/internal/app/azldev/cmds/image/boot.go b/internal/app/azldev/cmds/image/boot.go index fdad4bec..b09d991c 100644 --- a/internal/app/azldev/cmds/image/boot.go +++ b/internal/app/azldev/cmds/image/boot.go @@ -35,7 +35,7 @@ const ( defaultHostname = "azurelinux-vm" ) -// ImageFormat represents a bootable disk image format. +// ImageFormat represents a disk image or container image format. type ImageFormat string const ( @@ -47,12 +47,25 @@ const ( ImageFormatVhd ImageFormat = "vhd" // ImageFormatVhdx is the Hyper-V virtual hard disk format. ImageFormatVhdx ImageFormat = "vhdx" + // ImageFormatOCI is an OCI container image tarball. + ImageFormatOCI ImageFormat = "oci" ) -// SupportedImageFormats returns the list of supported bootable image formats in priority order. -// When multiple formats exist, the first match in this order is selected. -func SupportedImageFormats() []string { - return []string{string(ImageFormatRaw), string(ImageFormatQcow2), string(ImageFormatVhdx), string(ImageFormatVhd)} +// AllImageFormats returns all supported image formats in priority order. +func AllImageFormats() []string { + return []string{ + string(ImageFormatRaw), string(ImageFormatQcow2), + string(ImageFormatVhdx), string(ImageFormatVhd), + string(ImageFormatOCI), + } +} + +// BootableImageFormats returns the subset of image formats that can be booted in a VM. +func BootableImageFormats() []string { + return []string{ + string(ImageFormatRaw), string(ImageFormatQcow2), + string(ImageFormatVhdx), string(ImageFormatVhd), + } } // Assert that [ImageFormat] implements the [pflag.Value] interface. @@ -74,7 +87,7 @@ func (f *ImageFormat) Set(value string) error { case string(ImageFormatVhdx): *f = ImageFormatVhdx default: - return fmt.Errorf("unsupported image format %#q; supported: %v", value, SupportedImageFormats()) + return fmt.Errorf("unsupported image format %#q; supported: %v", value, BootableImageFormats()) } return nil @@ -252,7 +265,7 @@ func addBootFlags(cmd *cobra.Command, options *ImageBootOptions) { // Register shell completions for flags. _ = cmd.RegisterFlagCompletionFunc("format", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { - return SupportedImageFormats(), cobra.ShellCompDirectiveNoFileComp + return BootableImageFormats(), cobra.ShellCompDirectiveNoFileComp }) _ = cmd.RegisterFlagCompletionFunc("arch", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { @@ -449,7 +462,9 @@ func resolveDiskSource(env *azldev.Env, options *ImageBootOptions) (imagePath, i slog.String("format", imageFormat), ) case options.ImageName != "": - imagePath, imageFormat, err = findBootableImageArtifact(env, options.ImageName, string(options.Format)) + imagePath, imageFormat, err = findImageArtifact( + env, options.ImageName, string(options.Format), BootableImageFormats(), + ) if err != nil { return "", "", err } @@ -570,18 +585,21 @@ func createBootTempDir(env *azldev.Env) (string, error) { // Most formats have a single extension matching the format name, but vhd accepts // both .vhd and .vhdfixed since QEMU treats them identically. func fileExtensionsForFormat(format string) []string { - if format == string(ImageFormatVhd) { + switch format { + case string(ImageFormatVhd): return []string{"vhd", "vhdfixed"} + case string(ImageFormatOCI): + return []string{"oci.tar.xz", "oci.tar.gz", "oci.tar"} + default: + return []string{format} } - - return []string{format} } -// findBootableImageArtifact locates a bootable image artifact in the output directory for the -// given image name. If format is specified, only that format is searched. Otherwise, formats -// are searched in priority order (raw, qcow2, vhdx, vhd) and the first match is returned. -func findBootableImageArtifact( - env *azldev.Env, imageName, format string, +// findImageArtifact locates an image artifact in the output directory for the given image +// name. If format is specified, only that format is searched. Otherwise, searchFormats are +// searched in priority order and the first match is returned. +func findImageArtifact( + env *azldev.Env, imageName, format string, searchFormats []string, ) (imagePath, imageFormat string, err error) { // First validate the image exists in project configuration. _, err = ResolveImageByName(env, imageName) @@ -605,7 +623,7 @@ func findBootableImageArtifact( } // Determine which formats to search. - formatsToSearch := SupportedImageFormats() + formatsToSearch := searchFormats if format != "" { formatsToSearch = []string{format} } @@ -645,14 +663,23 @@ func findBootableImageArtifact( } return "", "", fmt.Errorf( - "no bootable image artifact found in %#q; supported formats: %v", - imageOutputDir, SupportedImageFormats(), + "no image artifact found in %#q; supported formats: %v", + imageOutputDir, searchFormats, ) } // InferImageFormat determines the image format from the file extension. // Returns an error if the extension does not match a supported format. func InferImageFormat(imagePath string) (string, error) { + lower := strings.ToLower(imagePath) + + // Check multi-part extensions first (e.g., ".oci.tar.xz"). + for _, ext := range []string{".oci.tar.xz", ".oci.tar.gz", ".oci.tar"} { + if strings.HasSuffix(lower, ext) { + return string(ImageFormatOCI), nil + } + } + ext := strings.ToLower(filepath.Ext(imagePath)) if ext == "" { return "", fmt.Errorf( @@ -668,11 +695,10 @@ func InferImageFormat(imagePath string) (string, error) { } // Validate the inferred format is supported. - supported := SupportedImageFormats() - if !lo.Contains(supported, format) { + if !lo.Contains(AllImageFormats(), format) { return "", fmt.Errorf( "unsupported image format %#q inferred from %#q; supported formats: %v", - format, imagePath, supported, + format, imagePath, AllImageFormats(), ) } diff --git a/internal/app/azldev/cmds/image/boot_test.go b/internal/app/azldev/cmds/image/boot_test.go index 431e313a..baf4fa06 100644 --- a/internal/app/azldev/cmds/image/boot_test.go +++ b/internal/app/azldev/cmds/image/boot_test.go @@ -206,13 +206,22 @@ func TestResolveImageWithAvailableList_Found(t *testing.T) { assert.Equal(t, "My test image", cfg.Description) } -func TestSupportedImageFormats(t *testing.T) { - formats := image.SupportedImageFormats() +func TestBootableImageFormats(t *testing.T) { + formats := image.BootableImageFormats() require.NotEmpty(t, formats) assert.Contains(t, formats, "raw") assert.Contains(t, formats, "qcow2") assert.Contains(t, formats, "vhdx") assert.Contains(t, formats, "vhd") + assert.NotContains(t, formats, "oci", "OCI is not a bootable format") +} + +func TestAllImageFormats(t *testing.T) { + formats := image.AllImageFormats() + require.NotEmpty(t, formats) + assert.Contains(t, formats, "raw") + assert.Contains(t, formats, "qcow2") + assert.Contains(t, formats, "oci") } func TestInferImageFormat(t *testing.T) { @@ -226,6 +235,9 @@ func TestInferImageFormat(t *testing.T) { {name: "vhd", path: "/path/to/image.vhd", expected: "vhd"}, {name: "vhdfixed", path: "/path/to/image.vhdfixed", expected: "vhd"}, {name: "vhdx", path: "/path/to/image.vhdx", expected: "vhdx"}, + {name: "oci.tar.xz", path: "/path/to/image.oci.tar.xz", expected: "oci"}, + {name: "oci.tar.gz", path: "/path/to/image.oci.tar.gz", expected: "oci"}, + {name: "oci.tar", path: "/path/to/image.oci.tar", expected: "oci"}, } for _, test := range tests { diff --git a/internal/app/azldev/cmds/image/pytestrunner.go b/internal/app/azldev/cmds/image/pytestrunner.go new file mode 100644 index 00000000..554446b8 --- /dev/null +++ b/internal/app/azldev/cmds/image/pytestrunner.go @@ -0,0 +1,356 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package image + +import ( + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +const ( + // pythonProgram is the Python interpreter used to create venvs and run pytest. + pythonProgram = "python3" + + // venvDirName is the name of the venv directory created under the azldev work dir. + venvDirName = "pytest-venv" + + // imagePlaceholder is the placeholder token for the image path. + imagePlaceholder = "{image-path}" + // imageNamePlaceholder is the placeholder token for the image name. + imageNamePlaceholder = "{image-name}" + // capabilitiesPlaceholder is the placeholder token for the comma-delimited capabilities. + capabilitiesPlaceholder = "{capabilities}" +) + +// RunPytestSuite runs a pytest-based test suite natively using a Python venv. +func RunPytestSuite( + env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, + imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, +) error { + pytestConfig := suiteConfig.Pytest + if pytestConfig == nil { + return fmt.Errorf("test suite %#q is missing pytest configuration", suiteConfig.Name) + } + + slog.Info("Running pytest test suite", + slog.String("name", suiteConfig.Name), + slog.String("working-dir", pytestConfig.WorkingDir), + slog.String("image-path", options.ImagePath), + ) + + // Validate that the working directory exists. + if pytestConfig.WorkingDir != "" { + workingDirExists, err := fileutils.DirExists(env.FS(), pytestConfig.WorkingDir) + if err != nil { + return fmt.Errorf("cannot access working directory %#q:\n%w", pytestConfig.WorkingDir, err) + } + + if !workingDirExists { + return fmt.Errorf("working directory not found: %#q", pytestConfig.WorkingDir) + } + } + + // Ensure python3 is available. + if err := prereqs.RequireExecutable(env, pythonProgram, nil); err != nil { + return fmt.Errorf("python3 is required to run pytest tests:\n%w", err) + } + + // Set up or reuse the venv. + venvDir, err := ensurePytestVenv(env, suiteConfig.Name, pytestConfig) + if err != nil { + return err + } + + // Build the pytest command: expand test paths, substitute placeholders in extra args. + pytestArgs := BuildNativePytestArgs(pytestConfig, imageConfig, options) + + slog.Info("Running pytest", slog.Any("args", pytestArgs)) + + venvPython := filepath.Join(venvDir, "bin", pythonProgram) + + cmdArgs := append([]string{"-m", "pytest"}, pytestArgs...) + + if env.Verbose() { + cmdArgs = append(cmdArgs, "--log-cli-level=DEBUG") + } + + pytestCmd := exec.CommandContext(env, venvPython, cmdArgs...) + pytestCmd.Dir = pytestConfig.WorkingDir + pytestCmd.Stdout = os.Stdout + pytestCmd.Stderr = os.Stderr + + cmd, err := env.Command(pytestCmd) + if err != nil { + return fmt.Errorf("failed to create pytest command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return fmt.Errorf("pytest run failed:\n%w", err) + } + + return nil +} + +// ensurePytestVenv creates or reuses a Python venv for the given test suite and installs +// dependencies according to the configured install mode. The venv is created under the +// project's work directory. +func ensurePytestVenv( + env *azldev.Env, testName string, pytestConfig *projectconfig.PytestConfig, +) (string, error) { + venvDir := filepath.Join(env.WorkDir(), venvDirName, testName) + + venvPython := filepath.Join(venvDir, "bin", pythonProgram) + + venvExists, err := fileutils.Exists(env.FS(), venvPython) + if err != nil { + return "", fmt.Errorf("cannot check venv at %#q:\n%w", venvDir, err) + } + + if !venvExists { + if err := createPythonVenv(env, venvDir); err != nil { + return "", err + } + } else { + slog.Info("Reusing existing Python venv", slog.String("path", venvDir)) + } + + // Install dependencies according to the configured mode. + if err := installPytestDependencies(env, venvPython, pytestConfig); err != nil { + return "", err + } + + return venvDir, nil +} + +// createPythonVenv creates a new Python virtual environment at venvDir. +func createPythonVenv(env *azldev.Env, venvDir string) error { + slog.Info("Creating Python venv", slog.String("path", venvDir)) + + venvCmd := exec.CommandContext(env, pythonProgram, "-m", "venv", venvDir) + venvCmd.Stdout = os.Stdout + venvCmd.Stderr = os.Stderr + + cmd, err := env.Command(venvCmd) + if err != nil { + return fmt.Errorf("failed to create venv command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return fmt.Errorf("failed to create Python venv at %#q:\n%w", venvDir, err) + } + + return nil +} + +// installPytestDependencies installs Python dependencies into the venv according to the +// configured [projectconfig.PytestInstallMode]. +func installPytestDependencies( + env *azldev.Env, venvPython string, pytestConfig *projectconfig.PytestConfig, +) error { + mode := pytestConfig.EffectiveInstallMode() + + if mode == projectconfig.PytestInstallNone { + slog.Info("Skipping dependency installation (install mode 'none')") + + return nil + } + + if pytestConfig.WorkingDir == "" { + slog.Debug("No working directory configured; skipping dependency installation") + + return nil + } + + switch mode { + case projectconfig.PytestInstallPyproject: + return installFromPyproject(env, venvPython, pytestConfig.WorkingDir) + case projectconfig.PytestInstallRequirements: + return installFromRequirements(env, venvPython, pytestConfig.WorkingDir) + case projectconfig.PytestInstallNone: + // Already handled above, but listed for exhaustiveness. + return nil + default: + return fmt.Errorf("unsupported install mode %#q", mode) + } +} + +// installFromPyproject installs dependencies from pyproject.toml using editable mode. +// If pyproject.toml is not found, a warning is logged and installation is skipped. +func installFromPyproject(env *azldev.Env, venvPython string, workingDir string) error { + pyprojectPath := filepath.Join(workingDir, "pyproject.toml") + + pyprojectExists, err := fileutils.Exists(env.FS(), pyprojectPath) + if err != nil { + return fmt.Errorf("cannot check for pyproject.toml at %#q:\n%w", pyprojectPath, err) + } + + if !pyprojectExists { + slog.Warn("No pyproject.toml found; skipping dependency installation", + slog.String("working-dir", workingDir), + ) + + return nil + } + + slog.Info("Installing dependencies from pyproject.toml", + slog.String("pyproject", pyprojectPath), + ) + + pipCmd := exec.CommandContext( + env, venvPython, "-m", "pip", "install", "--quiet", "-e", workingDir, + ) + pipCmd.Stdout = os.Stdout + pipCmd.Stderr = os.Stderr + + cmd, err := env.Command(pipCmd) + if err != nil { + return fmt.Errorf("failed to create pip install command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return fmt.Errorf("failed to install dependencies from %#q:\n%w", pyprojectPath, err) + } + + return nil +} + +// installFromRequirements installs dependencies from requirements.txt. +// Returns an error if the file is not found. +func installFromRequirements(env *azldev.Env, venvPython string, workingDir string) error { + requirementsPath := filepath.Join(workingDir, "requirements.txt") + + requirementsExists, err := fileutils.Exists(env.FS(), requirementsPath) + if err != nil { + return fmt.Errorf("cannot check for requirements.txt at %#q:\n%w", requirementsPath, err) + } + + if !requirementsExists { + return fmt.Errorf( + "requirements.txt not found at %#q (required by install mode %#q)", + requirementsPath, projectconfig.PytestInstallRequirements, + ) + } + + slog.Info("Installing dependencies from requirements.txt", + slog.String("requirements", requirementsPath), + ) + + pipCmd := exec.CommandContext( + env, venvPython, "-m", "pip", "install", "--quiet", "-r", requirementsPath, + ) + pipCmd.Stdout = os.Stdout + pipCmd.Stderr = os.Stderr + + cmd, err := env.Command(pipCmd) + if err != nil { + return fmt.Errorf("failed to create pip install command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return fmt.Errorf("failed to install dependencies from %#q:\n%w", requirementsPath, err) + } + + return nil +} + +// BuildNativePytestArgs constructs the full pytest argument list from the config. +// Test paths are glob-expanded relative to the working directory. Extra args are passed +// verbatim after placeholder substitution. The --junit-xml flag is appended automatically +// when requested via CLI. +func BuildNativePytestArgs( + pytestConfig *projectconfig.PytestConfig, + imageConfig *projectconfig.ImageConfig, + options *ImageTestOptions, +) []string { + absImagePath, err := filepath.Abs(options.ImagePath) + if err != nil { + absImagePath = options.ImagePath + } + + // Build a replacer for all known placeholders. + replacer := strings.NewReplacer( + imagePlaceholder, absImagePath, + imageNamePlaceholder, options.ImageName, + capabilitiesPlaceholder, strings.Join(imageConfig.Capabilities.EnabledNames(), ","), + ) + + args := make([]string, 0, len(pytestConfig.TestPaths)+len(pytestConfig.ExtraArgs)) + + // Expand test paths (glob patterns resolved relative to working dir). + for _, testPath := range pytestConfig.TestPaths { + args = append(args, expandGlob(testPath, pytestConfig.WorkingDir)...) + } + + // Substitute placeholders in extra args (never glob-expanded). + for _, arg := range pytestConfig.ExtraArgs { + args = append(args, replacer.Replace(arg)) + } + + // Append --junit-xml when requested via CLI. + if options.JUnitXMLPath != "" { + args = append(args, "--junit-xml", options.JUnitXMLPath) + } + + return args +} + +// expandGlob expands a glob pattern relative to workingDir using doublestar, which supports +// recursive ** patterns. If the pattern matches no files, the original pattern is returned +// unchanged (letting pytest report the error). +func expandGlob(pattern string, workingDir string) []string { + absPattern := pattern + if workingDir != "" && !filepath.IsAbs(pattern) { + absPattern = filepath.Join(workingDir, pattern) + } + + // Use WithFilesOnly so directory entries are excluded from glob results — pytest handles + // directory args directly (without globs). Use WithFailOnIOErrors to surface real I/O + // problems instead of silently returning empty. Follow symlinks (the default) since test + // trees may use them. + matches, err := doublestar.FilepathGlob(absPattern, + doublestar.WithFilesOnly(), + doublestar.WithFailOnIOErrors(), + ) + if err != nil { + slog.Warn("Failed to expand glob pattern", + slog.String("pattern", pattern), + slog.Any("error", err), + ) + + return []string{pattern} + } + + if len(matches) == 0 { + return []string{pattern} + } + + // Convert back to paths relative to the working directory so pytest sees them + // the same way it would with shell expansion. + result := make([]string, 0, len(matches)) + + for _, match := range matches { + if workingDir != "" { + rel, relErr := filepath.Rel(workingDir, match) + if relErr == nil { + result = append(result, rel) + + continue + } + } + + result = append(result, match) + } + + return result +} diff --git a/internal/app/azldev/cmds/image/pytestrunner_test.go b/internal/app/azldev/cmds/image/pytestrunner_test.go new file mode 100644 index 00000000..73b86b0e --- /dev/null +++ b/internal/app/azldev/cmds/image/pytestrunner_test.go @@ -0,0 +1,226 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package image_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/image" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testImageConfig returns a minimal [projectconfig.ImageConfig] for use in tests. +func testImageConfig() *projectconfig.ImageConfig { + return &projectconfig.ImageConfig{ + Name: "test-image", + } +} + +func boolPtr(v bool) *bool { + return &v +} + +func TestBuildNativePytestArgs_BasicTestPaths(t *testing.T) { + pytestConfig := &projectconfig.PytestConfig{ + TestPaths: []string{"cases/", "other/"}, + ExtraArgs: []string{"--image-path", "{image-path}"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + assert.Equal(t, []string{"cases/", "other/", "--image-path", "/images/test.raw"}, args) +} + +func TestBuildNativePytestArgs_GlobExpansion(t *testing.T) { + tmpDir := t.TempDir() + casesDir := filepath.Join(tmpDir, "cases") + require.NoError(t, os.MkdirAll(casesDir, 0o755)) + + for _, name := range []string{"test_alpha.py", "test_beta.py", "helper.py"} { + require.NoError(t, os.WriteFile(filepath.Join(casesDir, name), []byte("# test"), 0o600)) + } + + pytestConfig := &projectconfig.PytestConfig{ + WorkingDir: tmpDir, + TestPaths: []string{"cases/test_*.py"}, + ExtraArgs: []string{"--image-path", "{image-path}"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + assert.Contains(t, args, filepath.Join("cases", "test_alpha.py")) + assert.Contains(t, args, filepath.Join("cases", "test_beta.py")) + assert.NotContains(t, args, filepath.Join("cases", "helper.py")) + assert.Contains(t, args, "--image-path") + assert.Contains(t, args, "/images/test.raw") +} + +func TestBuildNativePytestArgs_GlobNoMatch(t *testing.T) { + tmpDir := t.TempDir() + + pytestConfig := &projectconfig.PytestConfig{ + WorkingDir: tmpDir, + TestPaths: []string{"cases/test_*.py"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + // Original pattern preserved when no matches. + assert.Equal(t, []string{"cases/test_*.py"}, args) +} + +func TestBuildNativePytestArgs_ExtraArgsNeverGlobExpanded(t *testing.T) { + pytestConfig := &projectconfig.PytestConfig{ + ExtraArgs: []string{"--pattern", "test_*.py"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + // Glob chars in extra-args should be passed verbatim. + assert.Equal(t, []string{"--pattern", "test_*.py"}, args) +} + +func TestBuildNativePytestArgs_JUnitXMLAppended(t *testing.T) { + pytestConfig := &projectconfig.PytestConfig{ + TestPaths: []string{"cases/"}, + ExtraArgs: []string{"--image-path", "{image-path}"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + JUnitXMLPath: "/output/results.xml", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + assert.Equal(t, []string{ + "cases/", + "--image-path", "/images/test.raw", + "--junit-xml", "/output/results.xml", + }, args) +} + +func TestBuildNativePytestArgs_NoJUnitXMLWhenNotRequested(t *testing.T) { + pytestConfig := &projectconfig.PytestConfig{ + TestPaths: []string{"cases/"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + assert.NotContains(t, args, "--junit-xml") +} + +func TestBuildNativePytestArgs_EmptyConfig(t *testing.T) { + pytestConfig := &projectconfig.PytestConfig{} + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + assert.Empty(t, args) +} + +func TestBuildNativePytestArgs_PlaceholderNotInTestPaths(t *testing.T) { + // {image-path} in test-paths should NOT be substituted (it's only for extra-args). + pytestConfig := &projectconfig.PytestConfig{ + TestPaths: []string{"{image-path}"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + assert.Equal(t, []string{"{image-path}"}, args) +} + +func TestBuildNativePytestArgs_ImageNamePlaceholder(t *testing.T) { + pytestConfig := &projectconfig.PytestConfig{ + ExtraArgs: []string{"--image-name", "{image-name}"}, + } + options := &image.ImageTestOptions{ + ImageName: "vm-base", + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, testImageConfig(), options) + + assert.Equal(t, []string{"--image-name", "vm-base"}, args) +} + +func TestBuildNativePytestArgs_CapabilitiesPlaceholder(t *testing.T) { + imgConfig := &projectconfig.ImageConfig{ + Name: "vm-base", + Capabilities: projectconfig.ImageCapabilities{ + MachineBootable: boolPtr(true), + Container: boolPtr(false), + Systemd: boolPtr(true), + RuntimePackageManagement: boolPtr(true), + }, + } + pytestConfig := &projectconfig.PytestConfig{ + ExtraArgs: []string{"--capabilities", "{capabilities}"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, imgConfig, options) + + assert.Equal(t, []string{"--capabilities", "machine-bootable,systemd,runtime-package-management"}, args) +} + +func TestBuildNativePytestArgs_CapabilitiesEmpty(t *testing.T) { + imgConfig := &projectconfig.ImageConfig{ + Name: "distroless", + Capabilities: projectconfig.ImageCapabilities{ + MachineBootable: boolPtr(false), + Container: boolPtr(true), + RuntimePackageManagement: boolPtr(false), + }, + } + pytestConfig := &projectconfig.PytestConfig{ + ExtraArgs: []string{"--capabilities", "{capabilities}"}, + } + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + args := image.BuildNativePytestArgs(pytestConfig, imgConfig, options) + + assert.Equal(t, []string{"--capabilities", "container"}, args) +} + +func TestRunPytestSuite_MissingPytestConfig(t *testing.T) { + suiteConfig := &projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + } + + options := &image.ImageTestOptions{ + ImagePath: "/images/test.raw", + } + + err := image.RunPytestSuite(nil, suiteConfig, testImageConfig(), options) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing pytest configuration") +} diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index 00f533a0..ed715d1d 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -7,31 +7,34 @@ import ( "errors" "fmt" "log/slog" - "os" - "os/exec" - "path/filepath" + "slices" + "sort" "strings" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/samber/lo" "github.com/spf13/cobra" ) -const ( - // testRunnerLisa is the only supported test runner. - testRunnerLisa = "lisa" - - // testImagePrefix is the prefix used for qcow2 images created during image testing. - testImagePrefix = "azldevtest" -) - // ImageTestOptions holds the options for the 'image test' command. type ImageTestOptions struct { - ImagePath string - TestRunner string - RunbookPath string - AdminPrivateKeyPath string + // ImageName is the name of the image (positional argument), used to look up its + // test suites and optionally resolve the image artifact path. + ImageName string + + // TestSuites optionally selects specific test suites to run. When empty, all test + // suites associated with the image are run. + TestSuites []string + + // ImagePath is an optional explicit path to the image file. When empty, the image + // artifact is resolved from the image name in the output directory. + ImagePath string + + // JUnitXMLPath is an optional path for writing JUnit XML output. + JUnitXMLPath string } func testOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { @@ -43,168 +46,207 @@ func NewImageTestCmd() *cobra.Command { options := &ImageTestOptions{} cmd := &cobra.Command{ - Use: "test", + Use: "test IMAGE_NAME", Short: "Run tests against an Azure Linux image", - Long: `Run tests against an Azure Linux image using a supported test runner. - -Currently only the LISA test runner is supported. The image must be in qcow2, -vhd, or vhdfixed format. If the image is in vhd/vhdfixed format it is -automatically converted to qcow2 before running the tests. - -Requirements: - - lisa (Installation instructions: https://github.com/microsoft/lisa/blob/main/INSTALL.md) - - runbook file (YAML format defining the tests to run: https://github.com/microsoft/lisa/blob/main/docs/Runbooks.md) - - qemu-img (for vhd/vhdfixed to qcow2 conversion, if needed)`, - Example: ` # Run LISA tests against a qcow2 image - azldev image test --image-path ./out/image.qcow2 --test-runner lisa --runbook-path ./runbooks/smoke.yml - - # Run LISA tests against a vhd image (auto-converted to qcow2) - azldev image test --image-path ./out/image.vhd --test-runner lisa --runbook-path ./runbooks/smoke.yml`, - RunE: azldev.RunFuncWithoutRequiredConfig(func(env *azldev.Env) (interface{}, error) { - return nil, testImage(env, options) + Long: `Run tests against an Azure Linux image using test suites defined in the +project configuration. + +Test suites are defined in the [test-suites] section of azldev.toml and referenced +by images via the [images.NAME.tests] subtable. Each test suite specifies a type +and framework-specific configuration in a matching subtable. + +By default, all test suites associated with the named image are run. Use +--test-suite to select specific suites (may be repeated). + +The image artifact can be specified explicitly with --image-path, or resolved +automatically from the image name in the output directory. + +For pytest tests, azldev creates a Python virtual environment, installs +dependencies from pyproject.toml in the working directory, and runs pytest +with the configured test paths and extra arguments. Use {image-path} in +extra-args to insert the image path. Glob patterns (including **) in +test-paths are expanded automatically.`, + Example: ` # Run all test suites for an image (artifact auto-resolved from output dir) + azldev image test vm-base + + # Run all test suites with an explicit image path + azldev image test vm-base --image-path ./out/images/vm-base/image.raw + + # Run a specific test suite + azldev image test vm-base --test-suite common-vm-checks + + # Run multiple specific test suites + azldev image test vm-base --test-suite common-vm-checks --test-suite vm-base-checks + + # Generate JUnit XML output + azldev image test vm-base --junit-xml results.xml`, + Args: cobra.ExactArgs(1), + RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { + options.ImageName = args[0] + + return nil, runImageTest(env, options) }), + ValidArgsFunction: generateImageNameCompletions, } + cmd.Flags().StringSliceVar(&options.TestSuites, "test-suite", nil, + "Name of a test suite to run (may be repeated; defaults to all suites for the image)") + cmd.Flags().StringVarP(&options.ImagePath, "image-path", "i", "", - "Path to the disk image file to test") - _ = cmd.MarkFlagRequired("image-path") + "Path to the disk image file (resolved from image name if not specified)") _ = cmd.MarkFlagFilename("image-path") - cmd.Flags().StringVar(&options.TestRunner, "test-runner", "", - "Test runner to use (currently only 'lisa' is supported)") - _ = cmd.MarkFlagRequired("test-runner") - - cmd.Flags().StringVarP(&options.RunbookPath, "runbook-path", "r", "", - "Path to the test runbook file") - _ = cmd.MarkFlagRequired("runbook-path") - _ = cmd.MarkFlagFilename("runbook-path") - - cmd.Flags().StringVarP(&options.AdminPrivateKeyPath, "admin-private-key-path", "k", "", - "Path to the admin SSH private key file passed to LISA") - _ = cmd.MarkFlagRequired("admin-private-key-path") - _ = cmd.MarkFlagFilename("admin-private-key-path") + cmd.Flags().StringVar(&options.JUnitXMLPath, "junit-xml", "", + "Path for writing JUnit XML output") + _ = cmd.MarkFlagFilename("junit-xml") return cmd } -// testImage implements the core logic for the 'image test' command. -func testImage(env *azldev.Env, options *ImageTestOptions) error { - // Check 1: validate test runner. - if err := CheckTestRunner(options.TestRunner); err != nil { - return err +// runImageTest resolves which test suites to run and dispatches each one. +func runImageTest(env *azldev.Env, options *ImageTestOptions) error { + cfg := env.Config() + if cfg == nil { + return errors.New("no project configuration loaded") } - // Check 2: verify lisa is installed. - if err := checkLisaInstalled(env); err != nil { + // Resolve the image config from the positional argument. + imageConfig, err := ResolveImageByName(env, options.ImageName) + if err != nil { return err } - // Check 3: validate admin private key path. - if err := validateFileExists(env.FS(), options.AdminPrivateKeyPath); err != nil { - return fmt.Errorf("--admin-private-key-path:\n%w", err) - } + // Resolve image path: explicit --image-path takes precedence, otherwise resolve + // from the image name in the output directory. + imagePath := options.ImagePath + if imagePath == "" { + var resolveErr error - // Check 4: resolve the image to a qcow2 path (converting if necessary). - qcow2Path, err := ResolveQcow2Image(env, options.ImagePath) - if err != nil { - return err - } + imagePath, _, resolveErr = findImageArtifact(env, options.ImageName, "", AllImageFormats()) + if resolveErr != nil { + return resolveErr + } - return runLisa(env, options.RunbookPath, qcow2Path, options.AdminPrivateKeyPath) -} + slog.Info("Resolved image artifact", + slog.String("image", options.ImageName), + slog.String("path", imagePath), + ) + } -// CheckTestRunner returns an error if the test runner is not supported. -func CheckTestRunner(runner string) error { - if !strings.EqualFold(runner, testRunnerLisa) { - return fmt.Errorf("test runner %#q is not supported; only %#q is supported at this time", runner, testRunnerLisa) + // Validate that the image file exists. + if err := validateFileExists(env.FS(), imagePath); err != nil { + return fmt.Errorf("image path:\n%w", err) } - return nil -} + options.ImagePath = imagePath + + // Determine which test suites to run. + suiteNames := resolveTestSuiteNames(imageConfig, options.TestSuites) -// checkLisaInstalled verifies that the lisa executable is available on the host. -func checkLisaInstalled(env *azldev.Env) error { - if !env.CommandInSearchPath(testRunnerLisa) { - return errors.New("'lisa' is not installed or not found in PATH; " + - "please install LISA before running image tests") + // Warn when explicitly requested suites are not referenced by the image config. + if len(options.TestSuites) > 0 { + warnUnassociatedSuites(options.ImageName, imageConfig, options.TestSuites) } - return nil -} + if len(suiteNames) == 0 { + slog.Warn("No test suites to run for image", slog.String("image", options.ImageName)) -// ResolveQcow2Image inspects the image at imagePath and returns a path to a qcow2 image. -// If the image is already qcow2 it is returned as-is. If it is vhd or vhdfixed it is -// converted to qcow2 in a temporary directory. Any other format is an error. -func ResolveQcow2Image(env *azldev.Env, imagePath string) (string, error) { - format, err := InferImageFormat(imagePath) - if err != nil { - return "", err + return nil } - switch format { - case string(ImageFormatQcow2): - slog.Info("Image is already in qcow2 format, using as-is", slog.String("path", imagePath)) + // Resolve and run each test suite, continuing past failures so all suites get a chance + // to run. Config/resolution errors abort immediately since they indicate a broken setup. + var testFailures []string - return imagePath, nil + for _, suiteName := range suiteNames { + suiteConfig, err := resolveTestSuiteByName(cfg, suiteName) + if err != nil { + return err + } - case string(ImageFormatVhd): - return convertToQcow2(env, imagePath) + if err := runTestSuite(env, suiteConfig, imageConfig, options); err != nil { + slog.Error("Test suite failed", + slog.String("suite", suiteName), + slog.Any("error", err), + ) - default: - return "", fmt.Errorf( - "image format %#q is not supported for testing; supported formats: qcow2, vhd, vhdfixed", - format, - ) + testFailures = append(testFailures, suiteName) + } } -} -// convertToQcow2 converts a vhd/vhdfixed disk image to qcow2 format using qemu-img and -// returns the path to the converted file. The converted image is written alongside the -// source file and is the caller's responsibility to clean up (or accept it as a leftover). -func convertToQcow2(env *azldev.Env, srcPath string) (string, error) { - if !env.CommandInSearchPath("qemu-img") { - return "", errors.New("'qemu-img' is not installed or not found in PATH; " + - "it is required to convert vhd/vhdfixed images to qcow2") + if len(testFailures) > 0 { + return fmt.Errorf("%d of %d test suite(s) failed: %s", + len(testFailures), len(suiteNames), strings.Join(testFailures, ", ")) } - baseName := strings.TrimSuffix(filepath.Base(srcPath), filepath.Ext(srcPath)) - destFileName := baseName + ".qcow2" - - // Write the converted image alongside the source file so the path is predictable. - destPath := filepath.Join(filepath.Dir(srcPath), testImagePrefix+"-"+destFileName) - - slog.Info("Converting image to qcow2", - slog.String("src", srcPath), - slog.String("dest", destPath), - ) - - if env.DryRun() { - slog.Info("Dry-run: would convert image to qcow2", - slog.String("src", srcPath), - slog.String("dest", destPath), - ) + return nil +} - return destPath, nil +// resolveTestSuiteNames determines which test suites to run. If explicit names are +// provided, they are used as-is. Otherwise, all test suites associated with the image +// are returned. +func resolveTestSuiteNames( + imageConfig *projectconfig.ImageConfig, explicitSuites []string, +) []string { + if len(explicitSuites) > 0 { + return explicitSuites } - convertCmd := exec.CommandContext( - env, "qemu-img", "convert", "-O", "qcow2", srcPath, destPath, - ) - convertCmd.Stdout = os.Stdout - convertCmd.Stderr = os.Stderr + return imageConfig.TestNames() +} - cmd, err := env.Command(convertCmd) - if err != nil { - return "", fmt.Errorf("failed to create qemu-img command:\n%w", err) +// warnUnassociatedSuites logs a warning for each explicitly requested test suite +// that is not referenced by the image's test configuration. +func warnUnassociatedSuites( + imageName string, imageConfig *projectconfig.ImageConfig, explicitSuites []string, +) { + imageTestNames := imageConfig.TestNames() + + for _, name := range explicitSuites { + if !slices.Contains(imageTestNames, name) { + slog.Warn("Test suite is not associated with image", + slog.String("suite", name), + slog.String("image", imageName), + ) + } } +} - if err = cmd.Run(env); err != nil { - return "", fmt.Errorf("failed to convert image %#q to qcow2:\n%w", srcPath, err) +// resolveTestSuiteByName looks up a test suite by name in the project configuration. +func resolveTestSuiteByName( + cfg *projectconfig.ProjectConfig, suiteName string, +) (*projectconfig.TestSuiteConfig, error) { + suiteConfig, ok := cfg.TestSuites[suiteName] + if !ok { + availableSuites := lo.Keys(cfg.TestSuites) + sort.Strings(availableSuites) + + if len(availableSuites) == 0 { + return nil, fmt.Errorf( + "test suite %#q not found; no test suites defined in project configuration", suiteName) + } + + return nil, fmt.Errorf( + "test suite %#q not found; available test suites: %s", + suiteName, strings.Join(availableSuites, ", "), + ) } - slog.Info("Conversion complete", slog.String("dest", destPath)) + return &suiteConfig, nil +} + +// runTestSuite dispatches a single test suite to the appropriate runner. +func runTestSuite( + env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, + imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, +) error { + switch suiteConfig.Type { + case projectconfig.TestTypePytest: + return RunPytestSuite(env, suiteConfig, imageConfig, options) - return destPath, nil + default: + return fmt.Errorf("unsupported test type %#q for test suite %#q", suiteConfig.Type, suiteConfig.Name) + } } // validateFileExists returns an error if the path does not point to an existing regular file. @@ -229,37 +271,3 @@ func validateFileExists(fs opctx.FS, path string) error { return nil } - -// runLisa executes `lisa -r -v "qcow2:"` and streams its -// stdout and stderr directly to the terminal. -func runLisa(env *azldev.Env, runbookPath, qcow2ImagePath, adminPrivateKeyPath string) error { - slog.Info("Running LISA tests", - slog.String("runbook", runbookPath), - slog.String("image", qcow2ImagePath), - ) - - args := []string{ - "-r", runbookPath, - "-v", "qcow2:" + qcow2ImagePath, - "-v", "admin_private_key_file:" + adminPrivateKeyPath, - } - - lisaCmd := exec.CommandContext( - env, - testRunnerLisa, - args..., - ) - lisaCmd.Stdout = os.Stdout - lisaCmd.Stderr = os.Stderr - - cmd, err := env.Command(lisaCmd) - if err != nil { - return fmt.Errorf("failed to create lisa command:\n%w", err) - } - - if err = cmd.Run(env); err != nil { - return fmt.Errorf("lisa test run failed:\n%w", err) - } - - return nil -} diff --git a/internal/app/azldev/cmds/image/test_test.go b/internal/app/azldev/cmds/image/test_test.go index bf1f3a2a..26ab6e2b 100644 --- a/internal/app/azldev/cmds/image/test_test.go +++ b/internal/app/azldev/cmds/image/test_test.go @@ -14,67 +14,14 @@ import ( func TestNewImageTestCmd(t *testing.T) { cmd := image.NewImageTestCmd() require.NotNil(t, cmd) - assert.Equal(t, "test", cmd.Use) + assert.Equal(t, "test IMAGE_NAME", cmd.Use) assert.Contains(t, cmd.Short, "test") } func TestNewImageTestCmd_Flags(t *testing.T) { cmd := image.NewImageTestCmd() + assert.NotNil(t, cmd.Flags().Lookup("test-suite")) assert.NotNil(t, cmd.Flags().Lookup("image-path")) - assert.NotNil(t, cmd.Flags().Lookup("test-runner")) - assert.NotNil(t, cmd.Flags().Lookup("runbook-path")) - assert.NotNil(t, cmd.Flags().Lookup("admin-private-key-path")) -} - -func TestCheckTestRunner_UnsupportedRunner(t *testing.T) { - err := image.CheckTestRunner("unsupported-runner") - require.Error(t, err) - assert.Contains(t, err.Error(), "not supported") - assert.Contains(t, err.Error(), "unsupported-runner") - assert.Contains(t, err.Error(), "lisa") -} - -func TestCheckTestRunner_Lisa(t *testing.T) { - err := image.CheckTestRunner("lisa") - require.NoError(t, err) -} - -func TestCheckTestRunner_LisaCaseInsensitive(t *testing.T) { - err := image.CheckTestRunner("LISA") - require.NoError(t, err) -} - -func TestResolveQcow2Image_UnsupportedFormats(t *testing.T) { - tests := []struct { - name string - imagePath string - wantErr string - }{ - { - name: "raw format", - imagePath: "/path/to/image.raw", - wantErr: "not supported for testing", - }, - { - name: "vhdx format", - imagePath: "/path/to/image.vhdx", - wantErr: "not supported for testing", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := image.ResolveQcow2Image(nil, tc.imagePath) - require.Error(t, err) - assert.Contains(t, err.Error(), tc.wantErr) - }) - } -} - -func TestResolveQcow2Image_UnknownExtension(t *testing.T) { - _, err := image.ResolveQcow2Image(nil, "/path/to/image.iso") - require.Error(t, err) - // InferImageFormat rejects iso before we even check format support. - assert.Contains(t, err.Error(), "unsupported image format") + assert.NotNil(t, cmd.Flags().Lookup("junit-xml")) } diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index 4a614c10..d3eaeb61 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -112,6 +112,15 @@ func (f ConfigFile) Validate() error { } } + // Validate test suite configurations. + for suiteName, suite := range f.TestSuites { + suite.Name = suiteName + + if err := suite.Validate(); err != nil { + return fmt.Errorf("invalid test suite %#q:\n%w", suiteName, err) + } + } + return nil } diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index 693d1aca..f38c50ef 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -275,16 +275,16 @@ func mergePackageGroups(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error // mergeTestSuites merges test suite definitions from a loaded config file into the // resolved config. Duplicate test suite names are not allowed. func mergeTestSuites(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { - for testName, test := range loadedCfg.TestSuites { - if _, ok := resolvedCfg.TestSuites[testName]; ok { - return fmt.Errorf("%w: test suite %#q", ErrDuplicateTestSuites, testName) + for suiteName, suite := range loadedCfg.TestSuites { + if _, ok := resolvedCfg.TestSuites[suiteName]; ok { + return fmt.Errorf("%w: test suite %#q", ErrDuplicateTestSuites, suiteName) } // Fill out fields not explicitly serialized. - test.Name = testName - test.SourceConfigFile = loadedCfg + suite.Name = suiteName + suite.SourceConfigFile = loadedCfg - resolvedCfg.TestSuites[testName] = test + resolvedCfg.TestSuites[suiteName] = *(suite.WithAbsolutePaths(loadedCfg.dir)) } return nil diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index c86e7b57..b4aa6dd1 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -798,30 +798,35 @@ rpm-channel = "devel" func TestLoadAndResolveProjectConfig_TestSuite(t *testing.T) { const configContents = ` [test-suites.smoke] +type = "pytest" description = "Smoke tests for images" -[test-suites.integration] -description = "Integration tests" +[test-suites.smoke.pytest] +working-dir = "tests" +test-paths = ["cases/test_*.py"] +extra-args = ["--image-path", "{image-path}"] ` + configDir := filepath.Dir(testConfigPath) + ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) require.NoError(t, err) - require.Len(t, config.TestSuites, 2) + require.Len(t, config.TestSuites, 1) + // Check pytest test. if assert.Contains(t, config.TestSuites, "smoke") { smokeTest := config.TestSuites["smoke"] assert.Equal(t, "smoke", smokeTest.Name) + assert.Equal(t, TestTypePytest, smokeTest.Type) assert.Equal(t, "Smoke tests for images", smokeTest.Description) - } - - if assert.Contains(t, config.TestSuites, "integration") { - integrationTest := config.TestSuites["integration"] - assert.Equal(t, "integration", integrationTest.Name) - assert.Equal(t, "Integration tests", integrationTest.Description) + require.NotNil(t, smokeTest.Pytest) + assert.Equal(t, filepath.Join(configDir, "tests"), smokeTest.Pytest.WorkingDir) + assert.Equal(t, []string{"cases/test_*.py"}, smokeTest.Pytest.TestPaths) + assert.Equal(t, []string{"--image-path", "{image-path}"}, smokeTest.Pytest.ExtraArgs) } } @@ -834,11 +839,17 @@ func TestLoadAndResolveProjectConfig_DuplicateTests(t *testing.T) { includes = ["include.toml"] [test-suites.smoke] -description = "Smoke tests" +type = "pytest" + +[test-suites.smoke.pytest] +test-paths = ["cases/"] `}, {"/project/include.toml", ` [test-suites.smoke] -description = "Other smoke tests" +type = "pytest" + +[test-suites.smoke.pytest] +test-paths = ["other/"] `}, } @@ -853,10 +864,42 @@ description = "Other smoke tests" require.ErrorIs(t, err, ErrDuplicateTestSuites) } +func TestLoadAndResolveProjectConfig_InvalidTestType(t *testing.T) { + const configContents = ` +[test-suites.bad] +type = "unsupported" +` + + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) + + _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnknownTestType) +} + +func TestLoadAndResolveProjectConfig_TestMissingRequiredField(t *testing.T) { + const configContents = ` +[test-suites.smoke] +type = "pytest" +# Missing [test-suites.smoke.pytest] subtable +` + + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) + + _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + require.Error(t, err) + assert.ErrorIs(t, err, ErrMissingTestField) +} + func TestLoadAndResolveProjectConfig_ImageWithValidTestRef(t *testing.T) { const configContents = ` [test-suites.smoke] -description = "Smoke tests" +type = "pytest" + +[test-suites.smoke.pytest] +test-paths = ["cases/"] [images.myimage] description = "Test image" @@ -991,3 +1034,45 @@ rpm-channel = "new-channel" assert.Equal(t, "new-channel", config.DefaultPackageConfig.Publish.EffectiveRPMChannel(), "rpm-channel should take precedence over the deprecated channel field") } + +func TestLoadAndResolveProjectConfig_TestSuiteInstallMode(t *testing.T) { + const configContents = ` +[test-suites.smoke] +type = "pytest" + +[test-suites.smoke.pytest] +working-dir = "tests" +install = "requirements" +test-paths = ["cases/"] +` + + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) + + config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + require.NoError(t, err) + + if assert.Contains(t, config.TestSuites, "smoke") { + smokeTest := config.TestSuites["smoke"] + require.NotNil(t, smokeTest.Pytest) + assert.Equal(t, PytestInstallRequirements, smokeTest.Pytest.Install) + assert.Equal(t, PytestInstallRequirements, smokeTest.Pytest.EffectiveInstallMode()) + } +} + +func TestLoadAndResolveProjectConfig_TestSuiteInvalidInstallMode(t *testing.T) { + const configContents = ` +[test-suites.smoke] +type = "pytest" + +[test-suites.smoke.pytest] +install = "invalid" +` + + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) + + _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidInstallMode) +} diff --git a/internal/projectconfig/project.go b/internal/projectconfig/project.go index 68b5759f..0728ebd6 100644 --- a/internal/projectconfig/project.go +++ b/internal/projectconfig/project.go @@ -106,13 +106,13 @@ func validatePackageGroupMembership(groups map[string]PackageGroupConfig) error // validateImageTestReferences checks that every test suite name in an image's // [ImageConfig.Tests.TestSuites] list corresponds to a defined entry in the top-level // TestSuites map. -func validateImageTestReferences(images map[string]ImageConfig, tests map[string]TestSuiteConfig) error { +func validateImageTestReferences(images map[string]ImageConfig, testSuites map[string]TestSuiteConfig) error { for imageName, image := range images { - for _, testName := range image.TestNames() { - if _, ok := tests[testName]; !ok { + for _, suiteName := range image.TestNames() { + if _, ok := testSuites[suiteName]; !ok { return fmt.Errorf( "%w: image %#q references test suite %#q, which is not defined in [test-suites]", - ErrUndefinedTestSuite, imageName, testName, + ErrUndefinedTestSuite, imageName, suiteName, ) } } diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index 44aa91c3..ec195696 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -3,13 +3,37 @@ package projectconfig -import "errors" +import ( + "errors" + "fmt" + + "dario.cat/mergo" +) + +// TestType indicates the type of test framework used to run a test suite. +type TestType string + +const ( + // TestTypePytest uses pytest to run static/offline validation checks. + TestTypePytest TestType = "pytest" +) var ( // ErrDuplicateTestSuites is returned when duplicate conflicting test suite definitions are found. ErrDuplicateTestSuites = errors.New("duplicate test suite") + // ErrUnknownTestType is returned for unrecognized test types. + ErrUnknownTestType = errors.New("unknown test type") + // ErrMissingTestField is returned when a required test config field is missing. + ErrMissingTestField = errors.New("missing required test field") // ErrUndefinedTestSuite is returned when an image references a test suite name that is not defined. ErrUndefinedTestSuite = errors.New("undefined test suite reference") + // ErrMismatchedTestSubtable is returned when a test config has a subtable that does not + // match its declared type. Currently only one test type (pytest) exists, so this cannot + // trigger yet. When adding a new test type with its own subtable field, add cross-checks + // in [TestSuiteConfig.Validate] to ensure only the matching subtable is populated. + ErrMismatchedTestSubtable = errors.New("mismatched test subtable") + // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. + ErrInvalidInstallMode = errors.New("invalid install mode") ) // TestSuiteConfig defines a named test suite. @@ -20,7 +44,149 @@ type TestSuiteConfig struct { // Description of the test suite. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test suite"` + // Type indicates the test framework to use. + Type TestType `toml:"type" json:"type" jsonschema:"required,enum=pytest,title=Type,description=Type of test framework (pytest)"` + + // Pytest holds pytest-specific configuration. Required when Type is "pytest". + Pytest *PytestConfig `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=Pytest-specific configuration (required when type is pytest)"` + // Reference to the source config file that this definition came from; not present // in serialized files. SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-"` } + +// PytestInstallMode specifies how Python dependencies are installed for a pytest suite. +type PytestInstallMode string + +const ( + // PytestInstallPyproject installs dependencies from pyproject.toml using editable mode. + // If pyproject.toml is not found, a warning is logged and installation is skipped. + // This is the default when [PytestConfig.Install] is not specified. + PytestInstallPyproject PytestInstallMode = "pyproject" + // PytestInstallRequirements installs dependencies from requirements.txt. + // Returns an error if requirements.txt is not found. + PytestInstallRequirements PytestInstallMode = "requirements" + // PytestInstallNone skips dependency installation entirely. + PytestInstallNone PytestInstallMode = "none" +) + +// PytestConfig holds configuration specific to pytest-based test suites. +type PytestConfig struct { + // WorkingDir is the directory to use as the current working directory when running pytest. + // Relative paths are resolved against the config file's directory. + WorkingDir string `toml:"working-dir,omitempty" json:"workingDir,omitempty" jsonschema:"title=Working directory,description=Directory to use as CWD when running pytest"` + + // TestPaths is the list of test file paths or directories to pass to pytest as positional + // arguments. Glob patterns (e.g., cases/test_*.py) are expanded relative to WorkingDir. + TestPaths []string `toml:"test-paths,omitempty" json:"testPaths,omitempty" jsonschema:"title=Test paths,description=Test file paths or directories passed to pytest. Glob patterns are expanded."` + + // ExtraArgs is the list of additional arguments to pass to pytest. These are passed + // verbatim after placeholder substitution. Use {image-path} as a placeholder for the + // image path, which will be substituted at runtime. + ExtraArgs []string `toml:"extra-args,omitempty" json:"extraArgs,omitempty" jsonschema:"title=Extra arguments,description=Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path."` + + // Install specifies how Python dependencies are installed into the venv before running + // pytest. Defaults to "pyproject" when not specified. + Install PytestInstallMode `toml:"install,omitempty" json:"install,omitempty" jsonschema:"enum=pyproject,enum=requirements,enum=none,title=Install mode,description=How to install Python dependencies: pyproject (default)\\, requirements\\, or none"` +} + +// Validate checks that the test suite config has valid type-specific required fields and that +// only the matching subtable is present. +func (t *TestSuiteConfig) Validate() error { + switch t.Type { + case TestTypePytest: + if t.Pytest == nil { + return fmt.Errorf("%w: test suite %#q of type %#q requires a [pytest] subtable", + ErrMissingTestField, t.Name, t.Type) + } + + if err := t.Pytest.Validate(); err != nil { + return fmt.Errorf("test suite %#q: %w", t.Name, err) + } + + // NOTE: When adding a new test type with its own subtable field (e.g., Lisa *LisaConfig), + // add a mismatch check here: + // if t.Lisa != nil { return fmt.Errorf("%w: ...", ErrMismatchedTestSubtable) } + // and add the symmetric check in the new type's case branch. + + default: + return fmt.Errorf("%w: %#q (test suite: %#q)", ErrUnknownTestType, t.Type, t.Name) + } + + return nil +} + +// Validate checks that the [PytestConfig] fields are valid. +func (p *PytestConfig) Validate() error { + if p.Install != "" && !p.Install.isValid() { + return fmt.Errorf( + "%w: %#q; allowed values: %#q, %#q, %#q (or omit for default %#q)", + ErrInvalidInstallMode, p.Install, + PytestInstallPyproject, PytestInstallRequirements, PytestInstallNone, + PytestInstallPyproject, + ) + } + + // When 'install' is explicitly set to a mode that requires a working directory, + // 'working-dir' must also be specified. + if p.Install != "" && p.Install != PytestInstallNone && p.WorkingDir == "" { + return fmt.Errorf( + "%w: 'working-dir' is required when 'install' is %#q", + ErrMissingTestField, p.Install, + ) + } + + return nil +} + +// EffectiveInstallMode returns the install mode, defaulting to [PytestInstallPyproject] when +// the field is not set. +func (p *PytestConfig) EffectiveInstallMode() PytestInstallMode { + if p.Install == "" { + return PytestInstallPyproject + } + + return p.Install +} + +// isValid returns whether the mode is a recognized [PytestInstallMode] value. +func (m PytestInstallMode) isValid() bool { + switch m { + case PytestInstallPyproject, PytestInstallRequirements, PytestInstallNone: + return true + default: + return false + } +} + +// MergeUpdatesFrom updates the test suite config with overrides present in other. +func (t *TestSuiteConfig) MergeUpdatesFrom(other *TestSuiteConfig) error { + err := mergo.Merge(t, other, mergo.WithOverride, mergo.WithAppendSlice) + if err != nil { + return fmt.Errorf("failed to merge test suite config:\n%w", err) + } + + return nil +} + +// WithAbsolutePaths returns a copy of the test suite config with relative file paths converted +// to absolute paths (relative to referenceDir). +func (t *TestSuiteConfig) WithAbsolutePaths(referenceDir string) *TestSuiteConfig { + result := &TestSuiteConfig{ + Name: t.Name, + Description: t.Description, + Type: t.Type, + SourceConfigFile: t.SourceConfigFile, + } + + if t.Pytest != nil { + result.Pytest = &PytestConfig{ + WorkingDir: makeAbsolute(referenceDir, t.Pytest.WorkingDir), + TestPaths: t.Pytest.TestPaths, + ExtraArgs: t.Pytest.ExtraArgs, + Install: t.Pytest.Install, + } + } + + return result +} diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index 28286ab1..64b80de6 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -66,6 +66,183 @@ func TestImageConfig_TestNames(t *testing.T) { }) } +func TestTestSuiteConfig_Validate(t *testing.T) { + t.Run("valid pytest config", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + WorkingDir: "tests", + TestPaths: []string{"cases/"}, + ExtraArgs: []string{"--image-path", "{image-path}"}, + }, + } + assert.NoError(t, testConfig.Validate()) + }) + + t.Run("valid pytest config with install mode", func(t *testing.T) { + for _, mode := range []projectconfig.PytestInstallMode{ + projectconfig.PytestInstallPyproject, + projectconfig.PytestInstallRequirements, + projectconfig.PytestInstallNone, + } { + t.Run(string(mode), func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + Install: mode, + WorkingDir: "tests", + }, + } + assert.NoError(t, testConfig.Validate()) + }) + } + }) + + t.Run("valid pytest config with empty install mode", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + WorkingDir: "tests", + }, + } + assert.NoError(t, testConfig.Validate()) + }) + + t.Run("invalid install mode", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + Install: "bad-mode", + }, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidInstallMode) + assert.Contains(t, err.Error(), "bad-mode") + }) + + t.Run("install mode requires working-dir", func(t *testing.T) { + for _, mode := range []projectconfig.PytestInstallMode{ + projectconfig.PytestInstallPyproject, + projectconfig.PytestInstallRequirements, + } { + t.Run(string(mode), func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + Install: mode, + // WorkingDir intentionally omitted. + }, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "working-dir") + }) + } + }) + + t.Run("install none without working-dir is valid", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + Install: projectconfig.PytestInstallNone, + }, + } + assert.NoError(t, testConfig.Validate()) + }) + + t.Run("default install without working-dir is valid", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + // Both Install and WorkingDir omitted — default auto-detect. + }, + } + assert.NoError(t, testConfig.Validate()) + }) + + t.Run("pytest missing subtable", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "[pytest]") + }) + + t.Run("unknown test type", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "bad", + Type: "unknown-type", + } + err := testConfig.Validate() + require.Error(t, err) + assert.ErrorIs(t, err, projectconfig.ErrUnknownTestType) + }) +} + +func TestPytestConfig_EffectiveInstallMode(t *testing.T) { + t.Run("default is pyproject", func(t *testing.T) { + cfg := &projectconfig.PytestConfig{} + assert.Equal(t, projectconfig.PytestInstallPyproject, cfg.EffectiveInstallMode()) + }) + + t.Run("explicit mode is preserved", func(t *testing.T) { + cfg := &projectconfig.PytestConfig{Install: projectconfig.PytestInstallNone} + assert.Equal(t, projectconfig.PytestInstallNone, cfg.EffectiveInstallMode()) + }) + + t.Run("requirements mode", func(t *testing.T) { + cfg := &projectconfig.PytestConfig{Install: projectconfig.PytestInstallRequirements} + assert.Equal(t, projectconfig.PytestInstallRequirements, cfg.EffectiveInstallMode()) + }) +} + +func TestTestSuiteConfig_MergeUpdatesFrom(t *testing.T) { + t.Run("merge overrides non-zero fields", func(t *testing.T) { + base := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + WorkingDir: "tests", + }, + } + other := projectconfig.TestSuiteConfig{ + Description: "Updated description", + } + require.NoError(t, base.MergeUpdatesFrom(&other)) + assert.Equal(t, "Updated description", base.Description) + assert.Equal(t, "tests", base.Pytest.WorkingDir) + }) + + t.Run("merge appends test-paths", func(t *testing.T) { + base := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + TestPaths: []string{"cases/"}, + }, + } + other := projectconfig.TestSuiteConfig{ + Pytest: &projectconfig.PytestConfig{ + TestPaths: []string{"extra/"}, + }, + } + require.NoError(t, base.MergeUpdatesFrom(&other)) + assert.Equal(t, []string{"cases/", "extra/"}, base.Pytest.TestPaths) + }) +} + func TestValidateTestSuiteReferences(t *testing.T) { t.Run("valid references", func(t *testing.T) { cfg := projectconfig.ProjectConfig{ @@ -78,6 +255,10 @@ func TestValidateTestSuiteReferences(t *testing.T) { TestSuites: map[string]projectconfig.TestSuiteConfig{ "smoke": { Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{ + WorkingDir: "tests", + }, }, }, Components: make(map[string]projectconfig.ComponentConfig), diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 86a1ca20..62bb6beb 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -790,6 +790,43 @@ "additionalProperties": false, "type": "object" }, + "PytestConfig": { + "properties": { + "working-dir": { + "type": "string", + "title": "Working directory", + "description": "Directory to use as CWD when running pytest" + }, + "test-paths": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Test paths", + "description": "Test file paths or directories passed to pytest. Glob patterns are expanded." + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path." + }, + "install": { + "type": "string", + "enum": [ + "pyproject", + "requirements", + "none" + ], + "title": "Install mode", + "description": "How to install Python dependencies: pyproject (default), requirements, or none" + } + }, + "additionalProperties": false, + "type": "object" + }, "ReleaseConfig": { "properties": { "calculation": { @@ -890,10 +927,26 @@ "type": "string", "title": "Description", "description": "Description of this test suite" + }, + "type": { + "type": "string", + "enum": [ + "pytest" + ], + "title": "Type", + "description": "Type of test framework (pytest)" + }, + "pytest": { + "$ref": "#/$defs/PytestConfig", + "title": "Pytest config", + "description": "Pytest-specific configuration (required when type is pytest)" } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "type" + ] }, "TestSuiteRef": { "properties": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 86a1ca20..62bb6beb 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -790,6 +790,43 @@ "additionalProperties": false, "type": "object" }, + "PytestConfig": { + "properties": { + "working-dir": { + "type": "string", + "title": "Working directory", + "description": "Directory to use as CWD when running pytest" + }, + "test-paths": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Test paths", + "description": "Test file paths or directories passed to pytest. Glob patterns are expanded." + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path." + }, + "install": { + "type": "string", + "enum": [ + "pyproject", + "requirements", + "none" + ], + "title": "Install mode", + "description": "How to install Python dependencies: pyproject (default), requirements, or none" + } + }, + "additionalProperties": false, + "type": "object" + }, "ReleaseConfig": { "properties": { "calculation": { @@ -890,10 +927,26 @@ "type": "string", "title": "Description", "description": "Description of this test suite" + }, + "type": { + "type": "string", + "enum": [ + "pytest" + ], + "title": "Type", + "description": "Type of test framework (pytest)" + }, + "pytest": { + "$ref": "#/$defs/PytestConfig", + "title": "Pytest config", + "description": "Pytest-specific configuration (required when type is pytest)" } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "type" + ] }, "TestSuiteRef": { "properties": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 86a1ca20..62bb6beb 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -790,6 +790,43 @@ "additionalProperties": false, "type": "object" }, + "PytestConfig": { + "properties": { + "working-dir": { + "type": "string", + "title": "Working directory", + "description": "Directory to use as CWD when running pytest" + }, + "test-paths": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Test paths", + "description": "Test file paths or directories passed to pytest. Glob patterns are expanded." + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to pytest. Use {image-path} as a placeholder for the image path." + }, + "install": { + "type": "string", + "enum": [ + "pyproject", + "requirements", + "none" + ], + "title": "Install mode", + "description": "How to install Python dependencies: pyproject (default), requirements, or none" + } + }, + "additionalProperties": false, + "type": "object" + }, "ReleaseConfig": { "properties": { "calculation": { @@ -890,10 +927,26 @@ "type": "string", "title": "Description", "description": "Description of this test suite" + }, + "type": { + "type": "string", + "enum": [ + "pytest" + ], + "title": "Type", + "description": "Type of test framework (pytest)" + }, + "pytest": { + "$ref": "#/$defs/PytestConfig", + "title": "Pytest config", + "description": "Pytest-specific configuration (required when type is pytest)" } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "type" + ] }, "TestSuiteRef": { "properties": {