From 6010c9c2e32a209b9791c39e8099551b056752bf Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:02:04 +0000 Subject: [PATCH 1/8] Add pydantic projects --- .github/workflows/ci.yml | 2 ++ .github/workflows/{diff-pr.yml => diff.yml} | 0 .github/zizmor.yml | 2 +- crates/karva_benchmark/Cargo.toml | 8 ++++++++ .../benches/pydantic_settings_walltime.rs | 18 ++++++++++++++++++ .../benches/pydantic_walltime.rs | 18 ++++++++++++++++++ crates/karva_test/src/real_world_projects.rs | 18 +++++++++++++++++- 7 files changed, 64 insertions(+), 2 deletions(-) rename .github/workflows/{diff-pr.yml => diff.yml} (100%) create mode 100644 crates/karva_benchmark/benches/pydantic_settings_walltime.rs create mode 100644 crates/karva_benchmark/benches/pydantic_walltime.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c79caef0..7bbcd4d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,8 @@ jobs: matrix: project: - affect_walltime + - pydantic_settings_walltime + - pydantic_settings - sqlmodel_walltime - typer_walltime diff --git a/.github/workflows/diff-pr.yml b/.github/workflows/diff.yml similarity index 100% rename from .github/workflows/diff-pr.yml rename to .github/workflows/diff.yml diff --git a/.github/zizmor.yml b/.github/zizmor.yml index cecafc94..ab63c8ab 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,7 +1,7 @@ rules: template-injection: ignore: - - "diff-pr.yml" + - "diff.yml" unpinned-uses: config: policies: diff --git a/crates/karva_benchmark/Cargo.toml b/crates/karva_benchmark/Cargo.toml index 6ca8a516..7236d9d3 100644 --- a/crates/karva_benchmark/Cargo.toml +++ b/crates/karva_benchmark/Cargo.toml @@ -19,6 +19,14 @@ harness = false name = "affect_walltime" harness = false +[[bench]] +name = "pydantic_settings_walltime" +harness = false + +[[bench]] +name = "pydantic_walltime" +harness = false + [[bench]] name = "sqlmodel_walltime" harness = false diff --git a/crates/karva_benchmark/benches/pydantic_settings_walltime.rs b/crates/karva_benchmark/benches/pydantic_settings_walltime.rs new file mode 100644 index 00000000..ef95efbb --- /dev/null +++ b/crates/karva_benchmark/benches/pydantic_settings_walltime.rs @@ -0,0 +1,18 @@ +use divan::{Bencher, bench}; +use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; +use karva_test::real_world_projects::PYDANTIC_SETTINGS_PROJECT; + +#[bench(sample_size = 4, sample_count = 5)] +fn pydantic_settings(bencher: Bencher) { + let benchmark = ProjectBenchmark::new(PYDANTIC_SETTINGS_PROJECT.clone()); + + bench_project(bencher, &benchmark); +} + +fn main() { + let benchmark = ProjectBenchmark::new(PYDANTIC_SETTINGS_PROJECT.clone()); + + warmup_project(&benchmark); + + divan::main(); +} diff --git a/crates/karva_benchmark/benches/pydantic_walltime.rs b/crates/karva_benchmark/benches/pydantic_walltime.rs new file mode 100644 index 00000000..e059ca5d --- /dev/null +++ b/crates/karva_benchmark/benches/pydantic_walltime.rs @@ -0,0 +1,18 @@ +use divan::{Bencher, bench}; +use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; +use karva_test::real_world_projects::PYDANTIC_PROJECT; + +#[bench(sample_size = 4, sample_count = 5)] +fn pydantic(bencher: Bencher) { + let benchmark = ProjectBenchmark::new(PYDANTIC_PROJECT.clone()); + + bench_project(bencher, &benchmark); +} + +fn main() { + let benchmark = ProjectBenchmark::new(PYDANTIC_PROJECT.clone()); + + warmup_project(&benchmark); + + divan::main(); +} diff --git a/crates/karva_test/src/real_world_projects.rs b/crates/karva_test/src/real_world_projects.rs index 5b22b645..66be251b 100644 --- a/crates/karva_test/src/real_world_projects.rs +++ b/crates/karva_test/src/real_world_projects.rs @@ -365,11 +365,27 @@ pub static PYDANTIC_SETTINGS_PROJECT: RealWorldProject<'static> = RealWorldProje python_version: PythonVersion::PY313, }; +pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { + name: "pydantic", + repository: "https://github.com/pydantic/pydantic", + commit: "ed67e3ebf7c9a55de75de0e8995dbce36551eaca", + paths: &["tests"], + dependencies: &[ + "pytest", + "typing-extensions", + "annotated-types", + "pydantic-core", + "typing-inspection", + ], + python_version: PythonVersion::PY313, +}; + pub fn all_projects() -> Vec<&'static RealWorldProject<'static>> { vec![ &AFFECT_PROJECT, + &PYDANTIC_SETTINGS_PROJECT, + &PYDANTIC_PROJECT, &SQLMODEL_PROJECT, &TYPER_PROJECT, - &PYDANTIC_SETTINGS_PROJECT, ] } From dad773d71da041eb83d986f3c2d6e5e1935590e0 Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:10:27 +0000 Subject: [PATCH 2/8] Try fix pydantic --- crates/karva_test/src/real_world_projects.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/karva_test/src/real_world_projects.rs b/crates/karva_test/src/real_world_projects.rs index 66be251b..feb19f1b 100644 --- a/crates/karva_test/src/real_world_projects.rs +++ b/crates/karva_test/src/real_world_projects.rs @@ -374,7 +374,7 @@ pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { "pytest", "typing-extensions", "annotated-types", - "pydantic-core", + "pydantic-core --only-binary :all:", "typing-inspection", ], python_version: PythonVersion::PY313, From 310268063ba535c5a08825c7e6e61aad01d75ac7 Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:13:34 +0000 Subject: [PATCH 3/8] Try fix pydantic --- crates/karva_test/src/real_world_projects.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/karva_test/src/real_world_projects.rs b/crates/karva_test/src/real_world_projects.rs index feb19f1b..6dacb241 100644 --- a/crates/karva_test/src/real_world_projects.rs +++ b/crates/karva_test/src/real_world_projects.rs @@ -374,7 +374,9 @@ pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { "pytest", "typing-extensions", "annotated-types", - "pydantic-core --only-binary :all:", + "pydantic-core", + "--only-binary", + ":all:", "typing-inspection", ], python_version: PythonVersion::PY313, From 41f7943db171655cac3dd7bc75d22fd4e2faade0 Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:19:11 +0000 Subject: [PATCH 4/8] Try fix pydantic --- crates/karva_test/src/real_world_projects.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/karva_test/src/real_world_projects.rs b/crates/karva_test/src/real_world_projects.rs index 6dacb241..59bd13d4 100644 --- a/crates/karva_test/src/real_world_projects.rs +++ b/crates/karva_test/src/real_world_projects.rs @@ -262,6 +262,7 @@ fn install_dependencies(checkout: &Checkout, venv_dir: Option) -> Resul let output = Command::new("uv") .args(["pip", "install", "--python", venv_path.to_str().unwrap()]) .args(checkout.project().dependencies) + .arg("--no-build") .output() .context("Failed to execute uv pip install command")?; @@ -375,8 +376,6 @@ pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { "typing-extensions", "annotated-types", "pydantic-core", - "--only-binary", - ":all:", "typing-inspection", ], python_version: PythonVersion::PY313, From 7b07e18d7e134d9990d4e8bdc3fbd4e80756915a Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:24:34 +0000 Subject: [PATCH 5/8] Try fix pydantic --- crates/karva_diff/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/karva_diff/src/main.rs b/crates/karva_diff/src/main.rs index e724881f..355572b3 100644 --- a/crates/karva_diff/src/main.rs +++ b/crates/karva_diff/src/main.rs @@ -98,6 +98,7 @@ fn run( let old_output = Command::new("uv") .arg("run") + .arg("--no-project") .arg("--no-sync") .arg(&args.old_karva_binary) .arg("test") @@ -126,6 +127,7 @@ fn run( let new_output = Command::new("uv") .arg("run") + .arg("--no-project") .arg("--no-sync") .arg(&args.new_karva_binary) .arg("test") From 56a0efe9fb3089be67ce349ea7bfa1447856f224 Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:31:37 +0000 Subject: [PATCH 6/8] Try fix pydantic --- crates/karva_diff/src/main.rs | 3 +-- crates/karva_test/src/real_world_projects.rs | 16 +--------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/crates/karva_diff/src/main.rs b/crates/karva_diff/src/main.rs index 355572b3..fcd5ef78 100644 --- a/crates/karva_diff/src/main.rs +++ b/crates/karva_diff/src/main.rs @@ -87,6 +87,7 @@ fn run( new_temp: &mut NamedTempFile, accumulation_temp: &mut NamedTempFile, ) -> Result<()> { + println!("testing {:?}", project.name); let installed_project = project.setup(true)?; let paths: Vec = installed_project @@ -99,7 +100,6 @@ fn run( let old_output = Command::new("uv") .arg("run") .arg("--no-project") - .arg("--no-sync") .arg(&args.old_karva_binary) .arg("test") .args(&paths) @@ -128,7 +128,6 @@ fn run( let new_output = Command::new("uv") .arg("run") .arg("--no-project") - .arg("--no-sync") .arg(&args.new_karva_binary) .arg("test") .args(&paths) diff --git a/crates/karva_test/src/real_world_projects.rs b/crates/karva_test/src/real_world_projects.rs index 59bd13d4..b0f181f2 100644 --- a/crates/karva_test/src/real_world_projects.rs +++ b/crates/karva_test/src/real_world_projects.rs @@ -90,10 +90,6 @@ impl<'a> Checkout<'a> { const fn project(&self) -> &RealWorldProject<'a> { &self.project } - - const fn project_root(&self) -> &Utf8PathBuf { - &self.path - } } pub struct InstalledProject<'a> { @@ -272,17 +268,7 @@ fn install_dependencies(checkout: &Checkout, venv_dir: Option) -> Resul String::from_utf8_lossy(&output.stderr) ); - let output = Command::new("uv") - .args(["pip", "install", "--python", venv_path.to_str().unwrap()]) - .arg(checkout.project_root()) - .output() - .context("Failed to execute uv pip install command")?; - - anyhow::ensure!( - output.status.success(), - "Package installation failed: {}", - String::from_utf8_lossy(&output.stderr) - ); + eprintln!("Installed dependencies successfully"); Ok(()) } From d8cf0b0d37d812c8dacbe5239711f536d779cd2b Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:40:24 +0000 Subject: [PATCH 7/8] Try fix pydantic --- crates/karva_test/src/real_world_projects.rs | 49 +++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/karva_test/src/real_world_projects.rs b/crates/karva_test/src/real_world_projects.rs index b0f181f2..cbb5b104 100644 --- a/crates/karva_test/src/real_world_projects.rs +++ b/crates/karva_test/src/real_world_projects.rs @@ -32,6 +32,8 @@ pub struct RealWorldProject<'a> { pub dependencies: &'a [&'a str], /// Python version to use pub python_version: PythonVersion, + /// Whether to pip install the project root + pub install_root: bool, } impl<'a> RealWorldProject<'a> { @@ -90,6 +92,10 @@ impl<'a> Checkout<'a> { const fn project(&self) -> &RealWorldProject<'a> { &self.project } + + const fn project_root(&self) -> &Utf8PathBuf { + &self.path + } } pub struct InstalledProject<'a> { @@ -268,6 +274,20 @@ fn install_dependencies(checkout: &Checkout, venv_dir: Option) -> Resul String::from_utf8_lossy(&output.stderr) ); + if checkout.project().install_root { + let output = Command::new("uv") + .args(["pip", "install", "--python", venv_path.to_str().unwrap()]) + .arg(checkout.project_root()) + .output() + .context("Failed to execute uv pip install command")?; + + anyhow::ensure!( + output.status.success(), + "Package installation failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + eprintln!("Installed dependencies successfully"); Ok(()) @@ -313,6 +333,7 @@ pub static AFFECT_PROJECT: RealWorldProject<'static> = RealWorldProject { paths: &["tests"], dependencies: &["pydantic", "pydantic-settings", "pytest"], python_version: PythonVersion::PY313, + install_root: true, }; pub static SQLMODEL_PROJECT: RealWorldProject<'static> = RealWorldProject { @@ -332,6 +353,7 @@ pub static SQLMODEL_PROJECT: RealWorldProject<'static> = RealWorldProject { "jinja2", ], python_version: PythonVersion::PY313, + install_root: false, }; pub static TYPER_PROJECT: RealWorldProject<'static> = RealWorldProject { @@ -339,8 +361,16 @@ pub static TYPER_PROJECT: RealWorldProject<'static> = RealWorldProject { repository: "https://github.com/fastapi/typer", commit: "a9a6595ad74ed59805c085f9d5369e666b955818", paths: &["tests"], - dependencies: &["click", "typing-extensions", "pytest", "coverage"], + dependencies: &[ + "click", + "typing-extensions", + "pytest", + "coverage", + "shellingham", + "rich", + ], python_version: PythonVersion::PY313, + install_root: false, }; pub static PYDANTIC_SETTINGS_PROJECT: RealWorldProject<'static> = RealWorldProject { @@ -348,8 +378,17 @@ pub static PYDANTIC_SETTINGS_PROJECT: RealWorldProject<'static> = RealWorldProje repository: "https://github.com/pydantic/pydantic-settings", commit: "ffb3ac673633e4f26a512b0fbf986d9bf8821a50", paths: &["tests"], - dependencies: &["pydantic", "pytest", "python-dotenv", "typing-extensions"], + dependencies: &[ + "pydantic", + "pytest", + "python-dotenv", + "typing-extensions", + "pytest-examples", + "pytest-mock", + "pyyaml", + ], python_version: PythonVersion::PY313, + install_root: false, }; pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { @@ -363,8 +402,14 @@ pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { "annotated-types", "pydantic-core", "typing-inspection", + "pytest-examples", + "pytest-mock", + "dirty-equals", + "jsonschema", + "pytz", ], python_version: PythonVersion::PY313, + install_root: false, }; pub fn all_projects() -> Vec<&'static RealWorldProject<'static>> { From b210bf967f4c25dcffe40461433a37a95f9ed128 Mon Sep 17 00:00:00 2001 From: Matthew Mckee Date: Tue, 2 Dec 2025 22:49:24 +0000 Subject: [PATCH 8/8] Fix walltime --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bbcd4d7..27ba083e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,7 @@ jobs: project: - affect_walltime - pydantic_settings_walltime - - pydantic_settings + - pydantic_walltime - sqlmodel_walltime - typer_walltime