diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..3c69aef3 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,7 @@ +# Configuration for the actionlint tool, which we run via prek +# to verify the correctness of the syntax in our GitHub Actions workflows. + +self-hosted-runner: + # Various runners we use that aren't recognized out-of-the-box by actionlint: + labels: + - codspeed-macro diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0f6833d..bd7e4f2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: benchmarks-walltime: name: "walltime benchmarks (${{ matrix.project }})" - runs-on: ubuntu-latest # This should be `codspeed-macro` + runs-on: codspeed-macro needs: [determine_changes, generate-walltime-benchmarks-matrix] if: ${{ (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }} @@ -188,10 +188,10 @@ jobs: with: persist-credentials: false - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - - name: "Install Rust toolchain" - run: rustup show + run: rustup update && rustup show + + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: "Install cargo codspeed" uses: taiki-e/install-action@a416ddeedbd372e614cc1386e8b642692f66865e # v2.57.1 diff --git a/crates/karva_benchmark/Cargo.toml b/crates/karva_benchmark/Cargo.toml index ffa0d4a3..28225b01 100644 --- a/crates/karva_benchmark/Cargo.toml +++ b/crates/karva_benchmark/Cargo.toml @@ -23,23 +23,7 @@ karva_runner = { workspace = true } karva_system = { workspace = true } [[bench]] -name = "affect_walltime" -harness = false - -[[bench]] -name = "pydantic_settings_walltime" -harness = false - -[[bench]] -name = "pydantic_walltime" -harness = false - -[[bench]] -name = "sqlmodel_walltime" -harness = false - -[[bench]] -name = "typer_walltime" +name = "karva_benchmark_walltime" harness = false [lints] diff --git a/crates/karva_benchmark/benches/affect_walltime.rs b/crates/karva_benchmark/benches/affect_walltime.rs deleted file mode 100644 index edb01646..00000000 --- a/crates/karva_benchmark/benches/affect_walltime.rs +++ /dev/null @@ -1,18 +0,0 @@ -use divan::{Bencher, bench}; -use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; -use karva_projects::real_world_projects::AFFECT_PROJECT; - -#[bench(sample_size = 4, sample_count = 5)] -fn affect(bencher: Bencher) { - let benchmark = ProjectBenchmark::new(AFFECT_PROJECT.clone()); - - bench_project(bencher, &benchmark); -} - -fn main() { - let benchmark = ProjectBenchmark::new(AFFECT_PROJECT.clone()); - - warmup_project(&benchmark); - - divan::main(); -} diff --git a/crates/karva_benchmark/benches/karva_benchmark_walltime.rs b/crates/karva_benchmark/benches/karva_benchmark_walltime.rs new file mode 100644 index 00000000..688c8ace --- /dev/null +++ b/crates/karva_benchmark/benches/karva_benchmark_walltime.rs @@ -0,0 +1,18 @@ +use divan::{Bencher, bench}; +use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; +use karva_projects::real_world_projects::KARVA_BENCHMARK_PROJECT; + +#[bench(sample_size = 3, sample_count = 3)] +fn karva_benchmark(bencher: Bencher) { + let benchmark = ProjectBenchmark::new(KARVA_BENCHMARK_PROJECT.clone()); + + bench_project(bencher, &benchmark); +} + +fn main() { + let benchmark = ProjectBenchmark::new(KARVA_BENCHMARK_PROJECT.clone()); + + warmup_project(&benchmark); + + divan::main(); +} diff --git a/crates/karva_benchmark/benches/pydantic_settings_walltime.rs b/crates/karva_benchmark/benches/pydantic_settings_walltime.rs deleted file mode 100644 index c069d3fc..00000000 --- a/crates/karva_benchmark/benches/pydantic_settings_walltime.rs +++ /dev/null @@ -1,18 +0,0 @@ -use divan::{Bencher, bench}; -use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; -use karva_projects::real_world_projects::PYDANTIC_SETTINGS_PROJECT; - -#[bench(sample_size = 4, sample_count = 5)] -fn pydantic_settings(bencher: Bencher) { - let benchmark = ProjectBenchmark::new(PYDANTIC_SETTINGS_PROJECT.clone()); - - bench_project(bencher, &benchmark); -} - -fn main() { - let benchmark = ProjectBenchmark::new(PYDANTIC_SETTINGS_PROJECT.clone()); - - warmup_project(&benchmark); - - divan::main(); -} diff --git a/crates/karva_benchmark/benches/pydantic_walltime.rs b/crates/karva_benchmark/benches/pydantic_walltime.rs deleted file mode 100644 index f989443f..00000000 --- a/crates/karva_benchmark/benches/pydantic_walltime.rs +++ /dev/null @@ -1,18 +0,0 @@ -use divan::{Bencher, bench}; -use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; -use karva_projects::real_world_projects::PYDANTIC_PROJECT; - -#[bench(sample_size = 1, sample_count = 3)] -fn pydantic(bencher: Bencher) { - let benchmark = ProjectBenchmark::new(PYDANTIC_PROJECT.clone()); - - bench_project(bencher, &benchmark); -} - -fn main() { - let benchmark = ProjectBenchmark::new(PYDANTIC_PROJECT.clone()); - - warmup_project(&benchmark); - - divan::main(); -} diff --git a/crates/karva_benchmark/benches/sqlmodel_walltime.rs b/crates/karva_benchmark/benches/sqlmodel_walltime.rs deleted file mode 100644 index 6c46a62e..00000000 --- a/crates/karva_benchmark/benches/sqlmodel_walltime.rs +++ /dev/null @@ -1,18 +0,0 @@ -use divan::{Bencher, bench}; -use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; -use karva_projects::real_world_projects::SQLMODEL_PROJECT; - -#[bench(sample_size = 3, sample_count = 4)] -fn sqlmodel(bencher: Bencher) { - let benchmark = ProjectBenchmark::new(SQLMODEL_PROJECT.clone()); - - bench_project(bencher, &benchmark); -} - -fn main() { - let benchmark = ProjectBenchmark::new(SQLMODEL_PROJECT.clone()); - - warmup_project(&benchmark); - - divan::main(); -} diff --git a/crates/karva_benchmark/benches/typer_walltime.rs b/crates/karva_benchmark/benches/typer_walltime.rs deleted file mode 100644 index 1dc0dcc2..00000000 --- a/crates/karva_benchmark/benches/typer_walltime.rs +++ /dev/null @@ -1,18 +0,0 @@ -use divan::{Bencher, bench}; -use karva_benchmark::walltime::{ProjectBenchmark, bench_project, warmup_project}; -use karva_projects::real_world_projects::TYPER_PROJECT; - -#[bench(sample_size = 1, sample_count = 3)] -fn typer(bencher: Bencher) { - let benchmark = ProjectBenchmark::new(TYPER_PROJECT.clone()); - - bench_project(bencher, &benchmark); -} - -fn main() { - let benchmark = ProjectBenchmark::new(TYPER_PROJECT.clone()); - - warmup_project(&benchmark); - - divan::main(); -} diff --git a/crates/karva_projects/src/real_world_projects.rs b/crates/karva_projects/src/real_world_projects.rs index 17064523..5a9787af 100644 --- a/crates/karva_projects/src/real_world_projects.rs +++ b/crates/karva_projects/src/real_world_projects.rs @@ -266,28 +266,27 @@ fn install_dependencies(checkout: &Checkout, venv_dir: Option) -> Resul "No dependencies to install for project '{}'", checkout.project().name ); - return Ok(()); - } - - let output = Command::new("uv") - .args([ - "pip", - "install", - "--python", - venv_path.to_str().unwrap(), - "--exclude-newer", - checkout.project().max_dep_date, - "--no-build", - ]) - .args(checkout.project().dependencies) - .output() - .context("Failed to execute uv pip install command")?; + } else { + let output = Command::new("uv") + .args([ + "pip", + "install", + "--python", + venv_path.to_str().unwrap(), + "--exclude-newer", + checkout.project().max_dep_date, + "--no-build", + ]) + .args(checkout.project().dependencies) + .output() + .context("Failed to execute uv pip install command")?; - anyhow::ensure!( - output.status.success(), - "Dependency installation failed: {}", - String::from_utf8_lossy(&output.stderr) - ); + anyhow::ensure!( + output.status.success(), + "Dependency installation failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } if let Ok(karva_wheel) = karva_system::find_karva_wheel() { let output = Command::new("uv") @@ -355,116 +354,19 @@ fn cargo_target_directory() -> Option<&'static PathBuf> { .as_ref() } -pub static AFFECT_PROJECT: RealWorldProject<'static> = RealWorldProject { - name: "affect", - repository: "https://github.com/MatthewMckee4/affect", - commit: "9a5198e46a5895ae3b3d56be80beb3bbb92c629e", +pub static KARVA_BENCHMARK_PROJECT: RealWorldProject<'static> = RealWorldProject { + name: "karva-benchmark-1", + repository: "https://github.com/karva-dev/karva-benchmark-1", + commit: "f099028b217f15311eae16f726ac12dabbeb87e9", paths: &["tests"], - dependencies: &["pydantic", "pydantic-settings", "pytest"], - max_dep_date: "2025-12-01", + dependencies: &[], + max_dep_date: "2026-12-01", python_version: PythonVersion::PY313, install_root: true, try_import_fixtures: false, retry: None, }; -pub static SQLMODEL_PROJECT: RealWorldProject<'static> = RealWorldProject { - name: "sqlmodel", - repository: "https://github.com/fastapi/sqlmodel", - commit: "43570910db2d7ab2e5efd96f60a0e2a3a61c5474", - paths: &["tests"], - dependencies: &[ - "pydantic", - "SQLAlchemy", - "pytest", - "dirty-equals", - "fastapi", - "httpx", - "coverage", - "black", - "jinja2", - ], - max_dep_date: "2025-12-01", - python_version: PythonVersion::PY313, - install_root: false, - try_import_fixtures: true, - retry: Some(3), -}; - -pub static TYPER_PROJECT: RealWorldProject<'static> = RealWorldProject { - name: "typer", - repository: "https://github.com/fastapi/typer", - commit: "a9a6595ad74ed59805c085f9d5369e666b955818", - paths: &["tests"], - dependencies: &[ - "click", - "typing-extensions", - "pytest", - "coverage", - "shellingham", - "rich", - ], - max_dep_date: "2025-12-01", - python_version: PythonVersion::PY313, - install_root: false, - try_import_fixtures: false, - retry: Some(3), -}; - -pub static PYDANTIC_SETTINGS_PROJECT: RealWorldProject<'static> = RealWorldProject { - name: "pydantic-settings", - repository: "https://github.com/pydantic/pydantic-settings", - commit: "ffb3ac673633e4f26a512b0fbf986d9bf8821a50", - paths: &["tests"], - dependencies: &[ - "pydantic", - "pytest", - "python-dotenv", - "typing-extensions", - "pytest-examples", - "pytest-mock", - "pyyaml", - ], - max_dep_date: "2025-12-01", - python_version: PythonVersion::PY313, - install_root: false, - try_import_fixtures: false, - retry: Some(3), -}; - -pub static PYDANTIC_PROJECT: RealWorldProject<'static> = RealWorldProject { - name: "pydantic", - // Skip recursive test that fails crashes karva and pytest. - repository: "https://github.com/MatthewMckee4/pydantic", - commit: "95068e360c8921db3b342e368154fab925fa299e", - paths: &["tests"], - dependencies: &[ - "pytest", - "typing-extensions", - "annotated-types", - "pydantic-core", - "typing-inspection", - "pytest-examples", - "pytest-mock", - "dirty-equals", - "jsonschema", - "pytz", - "hypothesis", - "inline_snapshot", - ], - max_dep_date: "2025-12-01", - python_version: PythonVersion::PY313, - install_root: false, - try_import_fixtures: true, - retry: Some(3), -}; - pub fn all_projects() -> Vec<&'static RealWorldProject<'static>> { - vec![ - &AFFECT_PROJECT, - &PYDANTIC_SETTINGS_PROJECT, - &PYDANTIC_PROJECT, - &SQLMODEL_PROJECT, - &TYPER_PROJECT, - ] + vec![&KARVA_BENCHMARK_PROJECT] }