diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c79caef0..27ba083e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,8 @@ jobs: matrix: project: - affect_walltime + - pydantic_settings_walltime + - pydantic_walltime - sqlmodel_walltime - typer_walltime diff --git a/.github/workflows/diff-pr.yml b/.github/workflows/diff.yml similarity index 100% rename from .github/workflows/diff-pr.yml rename to .github/workflows/diff.yml diff --git a/.github/zizmor.yml b/.github/zizmor.yml index cecafc94..ab63c8ab 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,7 +1,7 @@ rules: template-injection: ignore: - - "diff-pr.yml" + - "diff.yml" unpinned-uses: config: policies: diff --git a/crates/karva_benchmark/Cargo.toml b/crates/karva_benchmark/Cargo.toml index 6ca8a516..7236d9d3 100644 --- a/crates/karva_benchmark/Cargo.toml +++ b/crates/karva_benchmark/Cargo.toml @@ -19,6 +19,14 @@ harness = false name = "affect_walltime" harness = false +[[bench]] +name = "pydantic_settings_walltime" +harness = false + +[[bench]] +name = "pydantic_walltime" +harness = false + [[bench]] name = "sqlmodel_walltime" harness = false diff --git a/crates/karva_benchmark/benches/pydantic_settings_walltime.rs b/crates/karva_benchmark/benches/pydantic_settings_walltime.rs new file mode 100644 index 00000000..ef95efbb --- /dev/null +++ b/crates/karva_benchmark/benches/pydantic_settings_walltime.rs @@ -0,0 +1,18 @@ +use divan::{Bencher, bench}; +use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; +use karva_test::real_world_projects::PYDANTIC_SETTINGS_PROJECT; + +#[bench(sample_size = 4, sample_count = 5)] +fn pydantic_settings(bencher: Bencher) { + let benchmark = ProjectBenchmark::new(PYDANTIC_SETTINGS_PROJECT.clone()); + + bench_project(bencher, &benchmark); +} + +fn main() { + let benchmark = ProjectBenchmark::new(PYDANTIC_SETTINGS_PROJECT.clone()); + + warmup_project(&benchmark); + + divan::main(); +} diff --git a/crates/karva_benchmark/benches/pydantic_walltime.rs b/crates/karva_benchmark/benches/pydantic_walltime.rs new file mode 100644 index 00000000..e059ca5d --- /dev/null +++ b/crates/karva_benchmark/benches/pydantic_walltime.rs @@ -0,0 +1,18 @@ +use divan::{Bencher, bench}; +use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; +use karva_test::real_world_projects::PYDANTIC_PROJECT; + +#[bench(sample_size = 4, sample_count = 5)] +fn pydantic(bencher: Bencher) { + let benchmark = ProjectBenchmark::new(PYDANTIC_PROJECT.clone()); + + bench_project(bencher, &benchmark); +} + +fn main() { + let benchmark = ProjectBenchmark::new(PYDANTIC_PROJECT.clone()); + + warmup_project(&benchmark); + + divan::main(); +} diff --git a/crates/karva_diff/src/main.rs b/crates/karva_diff/src/main.rs index e724881f..fcd5ef78 100644 --- a/crates/karva_diff/src/main.rs +++ b/crates/karva_diff/src/main.rs @@ -87,6 +87,7 @@ fn run( new_temp: &mut NamedTempFile, accumulation_temp: &mut NamedTempFile, ) -> Result<()> { + println!("testing {:?}", project.name); let installed_project = project.setup(true)?; let paths: Vec = installed_project @@ -98,7 +99,7 @@ fn run( let old_output = Command::new("uv") .arg("run") - .arg("--no-sync") + .arg("--no-project") .arg(&args.old_karva_binary) .arg("test") .args(&paths) @@ -126,7 +127,7 @@ fn run( let new_output = Command::new("uv") .arg("run") - .arg("--no-sync") + .arg("--no-project") .arg(&args.new_karva_binary) .arg("test") .args(&paths) diff --git a/crates/karva_test/src/real_world_projects.rs b/crates/karva_test/src/real_world_projects.rs index 5b22b645..cbb5b104 100644 --- a/crates/karva_test/src/real_world_projects.rs +++ b/crates/karva_test/src/real_world_projects.rs @@ -32,6 +32,8 @@ pub struct RealWorldProject<'a> { pub dependencies: &'a [&'a str], /// Python version to use pub python_version: PythonVersion, + /// Whether to pip install the project root + pub install_root: bool, } impl<'a> RealWorldProject<'a> { @@ -262,6 +264,7 @@ fn install_dependencies(checkout: &Checkout, venv_dir: Option) -> Resul let output = Command::new("uv") .args(["pip", "install", "--python", venv_path.to_str().unwrap()]) .args(checkout.project().dependencies) + .arg("--no-build") .output() .context("Failed to execute uv pip install command")?; @@ -271,17 +274,21 @@ fn install_dependencies(checkout: &Checkout, venv_dir: Option) -> Resul String::from_utf8_lossy(&output.stderr) ); - let output = Command::new("uv") - .args(["pip", "install", "--python", venv_path.to_str().unwrap()]) - .arg(checkout.project_root()) - .output() - .context("Failed to execute uv pip install command")?; + if checkout.project().install_root { + let output = Command::new("uv") + .args(["pip", "install", "--python", venv_path.to_str().unwrap()]) + .arg(checkout.project_root()) + .output() + .context("Failed to execute uv pip install command")?; - anyhow::ensure!( - output.status.success(), - "Package installation failed: {}", - String::from_utf8_lossy(&output.stderr) - ); + anyhow::ensure!( + output.status.success(), + "Package installation failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + eprintln!("Installed dependencies successfully"); Ok(()) } @@ -326,6 +333,7 @@ pub static AFFECT_PROJECT: RealWorldProject<'static> = RealWorldProject { paths: &["tests"], dependencies: &["pydantic", "pydantic-settings", "pytest"], python_version: PythonVersion::PY313, + install_root: true, }; pub static SQLMODEL_PROJECT: RealWorldProject<'static> = RealWorldProject { @@ -345,6 +353,7 @@ pub static SQLMODEL_PROJECT: RealWorldProject<'static> = RealWorldProject { "jinja2", ], python_version: PythonVersion::PY313, + install_root: false, }; pub static TYPER_PROJECT: RealWorldProject<'static> = RealWorldProject { @@ -352,8 +361,16 @@ pub static TYPER_PROJECT: RealWorldProject<'static> = RealWorldProject { repository: "https://github.com/fastapi/typer", commit: "a9a6595ad74ed59805c085f9d5369e666b955818", paths: &["tests"], - dependencies: &["click", "typing-extensions", "pytest", "coverage"], + dependencies: &[ + "click", + "typing-extensions", + "pytest", + "coverage", + "shellingham", + "rich", + ], python_version: PythonVersion::PY313, + install_root: false, }; pub static PYDANTIC_SETTINGS_PROJECT: RealWorldProject<'static> = RealWorldProject { @@ -361,15 +378,46 @@ pub static PYDANTIC_SETTINGS_PROJECT: RealWorldProject<'static> = RealWorldProje repository: "https://github.com/pydantic/pydantic-settings", commit: "ffb3ac673633e4f26a512b0fbf986d9bf8821a50", paths: &["tests"], - dependencies: &["pydantic", "pytest", "python-dotenv", "typing-extensions"], + dependencies: &[ + "pydantic", + "pytest", + "python-dotenv", + "typing-extensions", + "pytest-examples", + "pytest-mock", + "pyyaml", + ], python_version: PythonVersion::PY313, + install_root: false, +}; + +pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { + name: "pydantic", + repository: "https://github.com/pydantic/pydantic", + commit: "ed67e3ebf7c9a55de75de0e8995dbce36551eaca", + paths: &["tests"], + dependencies: &[ + "pytest", + "typing-extensions", + "annotated-types", + "pydantic-core", + "typing-inspection", + "pytest-examples", + "pytest-mock", + "dirty-equals", + "jsonschema", + "pytz", + ], + python_version: PythonVersion::PY313, + install_root: false, }; pub fn all_projects() -> Vec<&'static RealWorldProject<'static>> { vec![ &AFFECT_PROJECT, + &PYDANTIC_SETTINGS_PROJECT, + &PYDANTIC_PROJECT, &SQLMODEL_PROJECT, &TYPER_PROJECT, - &PYDANTIC_SETTINGS_PROJECT, ] }