From 039cc59ac1e2a32da460a046f2b2e479c870d0fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 13:00:59 +0200 Subject: [PATCH 1/8] move overflow check --- .../rustc_trait_selection/src/solve/fulfill.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 4b80d22d6bb49..4554bcf38ea8e 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -209,12 +209,6 @@ where loop { let mut any_changed = false; for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) { - if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { - self.obligations.on_fulfillment_overflow(infcx); - // Only return true errors that we have accumulated while processing. - return errors; - } - let goal = obligation.as_goal(); let delegate = <&SolverDelegate<'tcx>>::from(infcx); if !delegate.disable_trait_solver_fast_paths() @@ -261,7 +255,14 @@ where // approximation and should only result in fulfillment overflow in // pathological cases. obligation.recursion_depth += 1; - any_changed = true; + + if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { + self.obligations.on_fulfillment_overflow(infcx); + // Only return true errors that we have accumulated while processing. + return errors; + } else { + any_changed = true; + } } match certainty { From 8cc6291e8691df7d69e61fa322be376d0da05631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 11:30:03 +0200 Subject: [PATCH 2/8] move fast path into evaluate_goal --- .../src/solve/eval_ctxt/mod.rs | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 05532cdb8924b..6b88ccfd623d2 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -591,6 +591,15 @@ where )); } + if !self.delegate.disable_trait_solver_fast_paths() + && let Some(certainty) = self.delegate.compute_goal_fast_path(goal, self.origin_span) + { + return Ok(( + NestedNormalizationGoals::empty(), + GoalEvaluation { goal, certainty, has_changed: HasChanged::No, stalled_on: None }, + )); + } + self.evaluate_goal_cold(source, goal) } @@ -1009,20 +1018,6 @@ where PredicateKind::NormalizesTo(_) )); - if !self.delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - self.delegate.compute_goal_fast_path(goal, self.origin_span) - { - match certainty { - Certainty::Yes => {} - Certainty::Maybe { .. } => { - self.nested_goals.push((source, goal, None)); - unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty)); - } - } - continue; - } - let GoalEvaluation { goal, certainty, has_changed, stalled_on } = self.evaluate_goal(source, goal, stalled_on)?; if has_changed == HasChanged::Yes { From 3536b417b53392757228279a335f89f8f7056358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 13:16:33 +0200 Subject: [PATCH 3/8] remove fastpath from fullfilment loop --- .../src/solve/fulfill.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 4554bcf38ea8e..0944da1f93dfe 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -7,7 +7,6 @@ use rustc_infer::traits::{ FromSolverError, PredicateObligation, PredicateObligations, TraitEngine, }; use rustc_middle::ty::{self, TyCtxt, TyVid, TypeVisitableExt, TypingMode}; -use rustc_next_trait_solver::delegate::SolverDelegate as _; use rustc_next_trait_solver::solve::{ GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _, StalledOnCoroutines, @@ -211,24 +210,6 @@ where for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) { let goal = obligation.as_goal(); let delegate = <&SolverDelegate<'tcx>>::from(infcx); - if !delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - delegate.compute_goal_fast_path(goal, obligation.cause.span) - { - match certainty { - // This fast path doesn't depend on region identity so it doesn't - // matter if the goal contains inference variables or not, so we - // don't need to call `push_hir_typeck_potentially_region_dependent_goal` - // here. - // - // Only goals proven via the trait solver should be region dependent. - Certainty::Yes => {} - Certainty::Maybe(_) => { - self.obligations.register(obligation, None); - } - } - continue; - } let result = delegate.evaluate_root_goal(goal, obligation.cause.span, stalled_on); self.inspect_evaluated_obligation(infcx, &obligation, &result); From 8a7f3276bd87ddb3e715aff36ded9d65eb5bd64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 14:12:54 +0200 Subject: [PATCH 4/8] experiment with better evaluate_goal_step --- compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 6b88ccfd623d2..d41454f5a0935 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -1011,6 +1011,9 @@ where ) -> Result, NoSolutionOrRerunNonErased> { // If this loop did not result in any progress, what's our final certainty. let mut unchanged_certainty = Some(Certainty::Yes); + // This mem::take seems super inefficient, given that we push to it again later. + // Despite that, replacing it has no effect on performance. We tried. + // (https://github.com/rust-lang/rust/pull/158126) for (source, goal, stalled_on) in mem::take(&mut self.nested_goals) { // We never handle `NormalizesTo` as a nested goal debug_assert!(!matches!( From 187954fb08d2d275043ff8d7a013b9534120c460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Mon, 22 Jun 2026 10:13:06 +0200 Subject: [PATCH 5/8] return stalled info from fast path --- compiler/rustc_middle/src/traits/solve.rs | 4 + .../rustc_next_trait_solver/src/delegate.rs | 5 +- .../src/solve/eval_ctxt/mod.rs | 200 +++++++++++------- .../rustc_next_trait_solver/src/solve/mod.rs | 20 +- .../src/solve/delegate.rs | 76 ++++--- compiler/rustc_type_ir/src/solve/mod.rs | 49 ++++- 6 files changed, 227 insertions(+), 127 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs index ceecb26d242dd..c45ebb80255cc 100644 --- a/compiler/rustc_middle/src/traits/solve.rs +++ b/compiler/rustc_middle/src/traits/solve.rs @@ -16,6 +16,10 @@ pub type CanonicalInput<'tcx, P = ty::Predicate<'tcx>> = ir::solve::CanonicalInp pub type CanonicalResponse<'tcx> = ir::solve::CanonicalResponse>; pub type FetchEligibleAssocItemResponse<'tcx> = ir::solve::FetchEligibleAssocItemResponse>; +pub type ComputeGoalFastPathOutcome<'tcx> = ir::solve::ComputeGoalFastPathOutcome>; +pub type GoalStalledOn<'tcx> = ir::solve::GoalStalledOn>; +pub type GoalStalledOnReason<'tcx> = ir::solve::GoalStalledOnReason>; +pub type SucceededInErased<'tcx> = ir::solve::SucceededInErased>; pub type PredefinedOpaques<'tcx> = &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>; diff --git a/compiler/rustc_next_trait_solver/src/delegate.rs b/compiler/rustc_next_trait_solver/src/delegate.rs index 8fd7d6d0471c7..949a42f694c4c 100644 --- a/compiler/rustc_next_trait_solver/src/delegate.rs +++ b/compiler/rustc_next_trait_solver/src/delegate.rs @@ -1,7 +1,8 @@ use std::ops::Deref; use rustc_type_ir::solve::{ - Certainty, FetchEligibleAssocItemResponse, Goal, NoSolution, VisibleForLeakCheck, + Certainty, ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, NoSolution, + VisibleForLeakCheck, }; use rustc_type_ir::{self as ty, InferCtxtLike, Interner, TypeFoldable}; @@ -23,7 +24,7 @@ pub trait SolverDelegate: Deref + Sized { &self, goal: Goal::Predicate>, span: ::Span, - ) -> Option; + ) -> ComputeGoalFastPathOutcome; fn fresh_var_for_kind_with_span( &self, diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index d41454f5a0935..1aa056adf2acb 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -10,9 +10,10 @@ use rustc_type_ir::relate::Relate; use rustc_type_ir::relate::solver_relating::RelateExt; use rustc_type_ir::search_graph::{CandidateHeadUsages, PathKind}; use rustc_type_ir::solve::{ - AccessedOpaques, ExternalRegionConstraints, FetchEligibleAssocItemResponse, MaybeInfo, - NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunCondition, - RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, + AccessedOpaques, ComputeGoalFastPathOutcome, ExternalRegionConstraints, + FetchEligibleAssocItemResponse, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, + QueryResultOrRerunNonErased, RerunCondition, RerunNonErased, RerunReason, RerunResultExt, + SmallCopyList, }; use rustc_type_ir::{ self as ty, CanonicalVarValues, ClauseKind, InferCtxtLike, Interner, MayBeErased, @@ -35,7 +36,7 @@ use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT, - Goal, GoalEvaluation, GoalSource, GoalStalledOn, HasChanged, MaybeCause, + Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnReason, HasChanged, MaybeCause, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, SucceededInErased, VisibleForLeakCheck, inspect, }; @@ -506,13 +507,8 @@ where } // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { - num_opaques, - ref stalled_vars, - ref sub_roots, - stalled_certainty, - ref previously_succeeded_in_erased, - }) = stalled_on + let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + stalled_on else { return MayMakeProgress; }; @@ -529,34 +525,46 @@ where return MayMakeProgress; } - // If any opaques changed in the opaque type storage, - // rerunning might make progress so we should rerun. - if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { - // Unless this goal previously succeeded in erased mode. - // If the stalled goal successfully evaluated while erasing opaque types, - // and the current state of the opaque type storage is not different in a way that is - // relevant, this stalled goal cannot make any progress and we set this variable to true. - let mut previous_erased_run_is_still_valid = false; - - if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased { - match self.should_rerun_after_erased_canonicalization( - accessed_opaques, - self.typing_mode(), - &self.delegate.clone_opaque_types_lookup_table(), - ) { - RerunDecision::Yes => {} - RerunDecision::EagerlyPropagateToParent => { - unreachable!("we never retry stalled queries if the parent was erased") + match reason { + GoalStalledOnReason::FastPath => { + // fastpath is never because of opaques, we can skip this check + } + &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + // If any opaques changed in the opaque type storage, + // rerunning might make progress so we should rerun. + if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) + { + // Unless this goal previously succeeded in erased mode. + // If the stalled goal successfully evaluated while erasing opaque types, + // and the current state of the opaque type storage is not different in a way that is + // relevant, this stalled goal cannot make any progress and we set this variable to true. + let mut previous_erased_run_is_still_valid = false; + + if let &SucceededInErased::Yes { accessed_opaques } = + previously_succeeded_in_erased + { + match self.should_rerun_after_erased_canonicalization( + accessed_opaques, + self.typing_mode(), + &self.delegate.clone_opaque_types_lookup_table(), + ) { + RerunDecision::Yes => {} + RerunDecision::EagerlyPropagateToParent => { + unreachable!( + "we never retry stalled queries if the parent was erased" + ) + } + RerunDecision::No => { + previous_erased_run_is_still_valid = true; + } + } } - RerunDecision::No => { - previous_erased_run_is_still_valid = true; + + if !previous_erased_run_is_still_valid { + return MayMakeProgress; } } } - - if !previous_erased_run_is_still_valid { - return MayMakeProgress; - } } // Otherwise, we can be sure that this stalled goal cannot make any progress @@ -564,6 +572,39 @@ where WontMakeProgress(stalled_certainty) } + /// This is a fast path optimization: + /// See the docs on [`ComputeGoalFastPathOutcome`] + pub fn compute_goal_fast_path( + &self, + goal: Goal, + ) -> Option<(NestedNormalizationGoals, GoalEvaluation)> { + if self.delegate.disable_trait_solver_fast_paths() { + return None; + } + + match self.delegate.compute_goal_fast_path(goal, self.origin_span) { + ComputeGoalFastPathOutcome::NoFastPath => None, + ComputeGoalFastPathOutcome::TriviallyHolds => Some(( + NestedNormalizationGoals::empty(), + GoalEvaluation { + goal, + certainty: Certainty::Yes, + has_changed: HasChanged::No, + stalled_on: None, + }, + )), + ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(( + NestedNormalizationGoals::empty(), + GoalEvaluation { + goal, + certainty: Certainty::AMBIGUOUS, + has_changed: HasChanged::No, + stalled_on: Some(stalled_on), + }, + )), + } + } + /// Recursively evaluates `goal`, returning the nested goals in case /// the nested goal is a `NormalizesTo` goal. /// @@ -591,13 +632,8 @@ where )); } - if !self.delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = self.delegate.compute_goal_fast_path(goal, self.origin_span) - { - return Ok(( - NestedNormalizationGoals::empty(), - GoalEvaluation { goal, certainty, has_changed: HasChanged::No, stalled_on: None }, - )); + if let Some(res) = self.compute_goal_fast_path(goal) { + return Ok(res); } self.evaluate_goal_cold(source, goal) @@ -782,41 +818,12 @@ where // that is not resolved. Only when *these* have changed is it meaningful // to recompute this goal. HasChanged::Yes => None, - HasChanged::No => { - // Remove the canonicalized universal vars, since we only care about stalled existentials. - let mut sub_roots = Vec::new(); - let mut stalled_vars = orig_values; - stalled_vars.retain(|arg| match arg.kind() { - // Lifetimes can never stall goals. - ty::GenericArgKind::Lifetime(_) => false, - ty::GenericArgKind::Type(ty) => match ty.kind() { - ty::Infer(ty::TyVar(vid)) => { - sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); - true - } - ty::Infer(_) => true, - ty::Param(_) | ty::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ty:?}"), - }, - ty::GenericArgKind::Const(ct) => match ct.kind() { - ty::ConstKind::Infer(_) => true, - ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ct:?}"), - }, - }); - - Some(GoalStalledOn { - num_opaques: canonical_goal - .canonical - .value - .predefined_opaques_in_body - .len(), - stalled_vars, - sub_roots, - stalled_certainty: certainty, - previously_succeeded_in_erased: succeeded_in_erased, - }) - } + HasChanged::No => Some(self.build_stalled_on( + canonical_goal, + certainty, + orig_values, + succeeded_in_erased, + )), }, }; @@ -826,6 +833,45 @@ where )) } + fn build_stalled_on( + &self, + canonical_goal: CanonicalInput, + certainty: Certainty, + mut stalled_vars: Vec, + previously_succeeded_in_erased: SucceededInErased, + ) -> GoalStalledOn { + // Remove the canonicalized universal vars, since we only care about stalled existentials. + let mut sub_roots = Vec::new(); + stalled_vars.retain(|arg| match arg.kind() { + // Lifetimes can never stall goals. + ty::GenericArgKind::Lifetime(_) => false, + ty::GenericArgKind::Type(ty) => match ty.kind() { + ty::Infer(ty::TyVar(vid)) => { + sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); + true + } + ty::Infer(_) => true, + ty::Param(_) | ty::Placeholder(_) => false, + _ => unreachable!("unexpected orig_value: {ty:?}"), + }, + ty::GenericArgKind::Const(ct) => match ct.kind() { + ty::ConstKind::Infer(_) => true, + ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => false, + _ => unreachable!("unexpected orig_value: {ct:?}"), + }, + }); + + GoalStalledOn { + stalled_vars, + sub_roots, + stalled_certainty: certainty, + reason: GoalStalledOnReason::Other { + num_opaques: canonical_goal.canonical.value.predefined_opaques_in_body.len(), + previously_succeeded_in_erased, + }, + } + } + fn should_rerun_after_erased_canonicalization( &self, AccessedOpaques { reason: _, rerun }: AccessedOpaques, diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index 27efb582dafcf..c1dad4c7a4aca 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -24,7 +24,7 @@ mod trait_goals; use derive_where::derive_where; use rustc_type_ir::inherent::*; pub use rustc_type_ir::solve::*; -use rustc_type_ir::{self as ty, Interner, TyVid}; +use rustc_type_ir::{self as ty, Interner}; use tracing::instrument; pub use self::eval_ctxt::{ @@ -425,21 +425,3 @@ pub struct GoalEvaluation { /// before rerunning it. pub stalled_on: Option>, } - -/// The conditions that must change for a goal to warrant -#[derive_where(Clone, Debug; I: Interner)] -pub struct GoalStalledOn { - pub num_opaques: usize, - pub stalled_vars: Vec, - pub sub_roots: Vec, - /// The certainty that will be returned on subsequent evaluations if this - /// goal remains stalled. - pub stalled_certainty: Certainty, - pub previously_succeeded_in_erased: SucceededInErased, -} - -#[derive_where(Clone, Debug; I: Interner)] -pub enum SucceededInErased { - Yes { accessed_opaques: AccessedOpaques }, - No, -} diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index f907b4ba19122..db65a6ae968cd 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -10,12 +10,15 @@ use rustc_infer::infer::canonical::{ QueryRegionConstraint, }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; -use rustc_infer::traits::solve::{FetchEligibleAssocItemResponse, Goal}; +use rustc_infer::traits::solve::{ + ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, +}; use rustc_middle::traits::query::NoSolution; use rustc_middle::traits::solve::Certainty; use rustc_middle::ty::{ self, MayBeErased, Ty, TyCtxt, TypeFlags, TypeFoldable, TypeVisitableExt, TypingMode, }; +use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnReason}; use rustc_span::{DUMMY_SP, Span}; use crate::traits::{EvaluateConstErr, ObligationCause, sizedness_fast_path, specialization_graph}; @@ -73,10 +76,25 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &self, goal: Goal<'tcx, ty::Predicate<'tcx>>, span: Span, - ) -> Option { + ) -> ComputeGoalFastPathOutcome<'tcx> { + use ComputeGoalFastPathOutcome as Outcome; + + fn stalled_with_args<'tcx>( + stalled_vars: Vec>, + ) -> ComputeGoalFastPathOutcome<'tcx> { + Outcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + reason: GoalStalledOnReason::FastPath, + }, + } + } + // FIXME(-Zassumptions-on-binders): actually handle fast path if self.tcx.assumptions_on_binders() { - return None; + return Outcome::NoFastPath; } let pred = goal.predicate.kind(); @@ -84,22 +102,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< ty::PredicateKind::Clause(ty::ClauseKind::Trait(trait_pred)) => { let trait_pred = pred.rebind(trait_pred); - if self.shallow_resolve(trait_pred.self_ty().skip_binder()).is_ty_var() + let self_ty = self.shallow_resolve(trait_pred.self_ty().skip_binder()); + if self_ty.is_ty_var() // We don't do this fast path when opaques are defined since we may // eventually use opaques to incompletely guide inference via ty var // self types. // FIXME: Properly consider opaques here. && self.known_no_opaque_types_in_storage() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![self_ty.into()]) } else if trait_pred.polarity() == ty::PredicatePolarity::Positive { match self.0.tcx.as_lang_item(trait_pred.def_id()) { Some(LangItem::Sized) | Some(LangItem::MetaSized) => { let predicate = self.resolve_vars_if_possible(goal.predicate); if sizedness_fast_path(self.tcx, predicate, goal.param_env) { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } Some(LangItem::Copy | LangItem::Clone) => { @@ -114,23 +133,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< .has_type_flags(TypeFlags::HAS_FREE_REGIONS | TypeFlags::HAS_INFER) && self_ty.is_trivially_pure_clone_copy() { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } else { - None + Outcome::NoFastPath } } ty::PredicateKind::DynCompatible(def_id) if self.0.tcx.is_dyn_compatible(def_id) => { - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.sub_regions( @@ -139,11 +158,11 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< outlives.0, ty::VisibleForLeakCheck::Yes, ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.register_type_outlives_constraint( @@ -152,48 +171,49 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &ObligationCause::dummy_with_span(span), ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Subtype(ty::SubtypePredicate { a, b, .. }) | ty::PredicateKind::Coerce(ty::CoercePredicate { a, b }) => { if a.has_escaping_bound_vars() || b.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } match (self.shallow_resolve(a).kind(), self.shallow_resolve(b).kind()) { (&ty::Infer(ty::TyVar(a_vid)), &ty::Infer(ty::TyVar(b_vid))) => { self.sub_unify_ty_vids_raw(a_vid, b_vid); - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![a.into(), b.into()]) } - _ => None, + _ => Outcome::NoFastPath, } } ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(ct, _)) => { if ct.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } - if self.shallow_resolve_const(ct).is_ct_infer() { - Some(Certainty::AMBIGUOUS) + let arg = self.shallow_resolve_const(ct); + if arg.is_ct_infer() { + stalled_with_args(vec![arg.into()]) } else { - None + Outcome::NoFastPath } } ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(arg)) => { if arg.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } let arg = self.shallow_resolve_term(arg); if arg.is_trivially_wf(self.tcx) { - Some(Certainty::Yes) + Outcome::TriviallyHolds } else if arg.is_infer() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![arg.into_arg()]) } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 8866c763927b3..5b07a23387ecb 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -16,7 +16,7 @@ use crate::lang_items::SolverTraitLangItem; use crate::region_constraint::RegionConstraint; use crate::search_graph::PathKind; use crate::{ - self as ty, Canonical, CanonicalVarValues, CantBeErased, Interner, TypingMode, Upcast, + self as ty, Canonical, CanonicalVarValues, CantBeErased, Interner, TyVid, TypingMode, Upcast, }; pub type CanonicalInput::Predicate> = @@ -942,3 +942,50 @@ impl SizedTraitKind { }) } } + +#[derive_where(Clone, Debug; I: Interner)] +pub enum SucceededInErased { + /// This goal previously succeeded in erased mode, which based on `accessed_opaques` + /// might make us take a fast path slightly more often. + Yes { + accessed_opaques: AccessedOpaques, + }, + No, +} + +#[derive_where(Clone, Debug; I: Interner)] +pub enum GoalStalledOnReason { + /// This goal got stalled in `compute_goal_fast_path`. Usually this means + /// the goal is stalled on not that much, only one or two variables, and + /// definitely nothing to do with opaque types. So we don't store that information. + FastPath, + Other { + num_opaques: usize, + previously_succeeded_in_erased: SucceededInErased, + }, +} + +/// The conditions that must change for a goal to warrant +#[derive_where(Clone, Debug; I: Interner)] +pub struct GoalStalledOn { + pub stalled_vars: Vec, + pub sub_roots: Vec, + /// The certainty that will be returned on subsequent evaluations if this + /// goal remains stalled. + pub stalled_certainty: Certainty, + pub reason: GoalStalledOnReason, +} + +/// For some goals we can trivially answer some questions without going through +/// canonicalization. There are three options: + +#[derive(Clone, Debug)] +pub enum ComputeGoalFastPathOutcome { + /// Do not attempt the fast path. Compute as normal. + NoFastPath, + /// The goal trivially holds, immediately produce a result with [`Certainty::Yes`] + TriviallyHolds, + /// The goal is trivially stalled: we know for sure that it makes no sense to compute it right + /// now, but can return information about what its stalled on and when it can be computed for real. + TriviallyStalled { stalled_on: GoalStalledOn }, +} From 4946875445c309d674e2500c85b1c72809f05e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Mon, 22 Jun 2026 14:19:47 +0200 Subject: [PATCH 6/8] move goal fast path into evaluate_goal_cold --- .../rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 1aa056adf2acb..a72b5a20e591e 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -632,10 +632,6 @@ where )); } - if let Some(res) = self.compute_goal_fast_path(goal) { - return Ok(res); - } - self.evaluate_goal_cold(source, goal) } @@ -646,6 +642,10 @@ where source: GoalSource, goal: Goal, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { + if let Some(res) = self.compute_goal_fast_path(goal) { + return Ok(res); + } + // We only care about one entry per `OpaqueTypeKey` here, // so we only canonicalize the lookup table and ignore // duplicate entries. From 58f22e89416534c46fde0248cb997ace6c0f662a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 25 Jun 2026 10:25:23 +0200 Subject: [PATCH 7/8] shuffle fastpaths around to before creating an evalctxt --- .../src/solve/eval_ctxt/fast_path.rs | 139 ++++++ .../src/solve/eval_ctxt/mod.rs | 398 +++++++----------- .../src/solve/project_goals/mod.rs | 2 +- 3 files changed, 285 insertions(+), 254 deletions(-) create mode 100644 compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs new file mode 100644 index 0000000000000..978d97f184797 --- /dev/null +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -0,0 +1,139 @@ +//! This file contains a number of standalone functions useful for taking _fast paths_ in the trait +//! solver. The exact place where we check for these fast paths changes, and matters a lot for +//! performance. Ideally we'd only check them in `evaluate_goal`, but when evaluating root goals +//! we can check them earlier and save some time creating an `EvalCtxt` in the first place. +//! +//! For debugging, fast paths can be disabled using `-Zdisable-fast-paths`. + +use rustc_type_ir::InferCtxtLike; +use rustc_type_ir::Interner; +use rustc_type_ir::inherent::*; +use rustc_type_ir::solve::Goal; +use rustc_type_ir::solve::{ + Certainty, ComputeGoalFastPathOutcome, GoalStalledOn, GoalStalledOnReason, SucceededInErased, +}; + +use crate::delegate::SolverDelegate; +use crate::solve::GoalEvaluation; +use crate::solve::HasChanged; +use crate::solve::eval_ctxt::RerunDecision; +use crate::solve::eval_ctxt::should_rerun_after_erased_canonicalization; + +#[derive(Debug, Clone, Copy)] +pub(super) enum RerunStalled { + WontMakeProgress(Certainty), + MayMakeProgress, +} + +/// If we have run a goal before, and it was stalled, check that any of the goal's +/// args have changed. This is a cheap way to determine that if we were to rerun this goal now, +/// it will remain stalled since it'll canonicalize the same way and evaluation is pure. +/// Therefore, we can skip this rerun +pub(super) fn rerunning_stalled_goal_may_make_progress( + delegate: &D, + stalled_on: Option<&GoalStalledOn>, +) -> RerunStalled +where + D: SolverDelegate, + I: Interner, +{ + use RerunStalled::*; + + // If fast paths are turned off, then we assume all goals can always make progress + if delegate.disable_trait_solver_fast_paths() { + return MayMakeProgress; + } + + // If the goal isn't stalled, we should definitely run it. + let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + stalled_on + else { + return MayMakeProgress; + }; + + // If any of the stalled goal's generic arguments changed, + // rerunning might make progress so we should rerun. + if stalled_vars.iter().any(|value| delegate.is_changed_arg(*value)) { + return MayMakeProgress; + } + + // If some inference took place in any of the sub roots, + // rerunning might make progress so we should rerun. + if sub_roots.iter().any(|&vid| delegate.sub_unification_table_root_var(vid) != vid) { + return MayMakeProgress; + } + + match reason { + GoalStalledOnReason::FastPath => { + // fastpath is never because of opaques, we can skip this check + } + &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + // If any opaques changed in the opaque type storage, + // rerunning might make progress so we should rerun. + if delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { + // Unless this goal previously succeeded in erased mode. + // If the stalled goal successfully evaluated while erasing opaque types, + // and the current state of the opaque type storage is not different in a way that is + // relevant, this stalled goal cannot make any progress and we set this variable to true. + let mut previous_erased_run_is_still_valid = false; + + if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased + { + match should_rerun_after_erased_canonicalization( + accessed_opaques, + delegate.typing_mode_raw(), + &delegate.clone_opaque_types_lookup_table(), + ) { + RerunDecision::Yes => {} + RerunDecision::EagerlyPropagateToParent => { + unreachable!("we never retry stalled queries if the parent was erased") + } + RerunDecision::No => { + previous_erased_run_is_still_valid = true; + } + } + } + + if !previous_erased_run_is_still_valid { + return MayMakeProgress; + } + } + } + } + + // Otherwise, we can be sure that this stalled goal cannot make any progress + // and we can exit early. + WontMakeProgress(stalled_certainty) +} + +/// This is a fast path optimization: +/// See the docs on [`ComputeGoalFastPathOutcome`] +pub(super) fn compute_goal_fast_path( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + if delegate.disable_trait_solver_fast_paths() { + return None; + } + + match delegate.compute_goal_fast_path(goal, origin_span) { + ComputeGoalFastPathOutcome::NoFastPath => None, + ComputeGoalFastPathOutcome::TriviallyHolds => Some(GoalEvaluation { + goal, + certainty: Certainty::Yes, + has_changed: HasChanged::No, + stalled_on: None, + }), + ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(GoalEvaluation { + goal, + certainty: Certainty::AMBIGUOUS, + has_changed: HasChanged::No, + stalled_on: Some(stalled_on), + }), + } +} diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index a72b5a20e591e..db5634a796611 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -10,10 +10,9 @@ use rustc_type_ir::relate::Relate; use rustc_type_ir::relate::solver_relating::RelateExt; use rustc_type_ir::search_graph::{CandidateHeadUsages, PathKind}; use rustc_type_ir::solve::{ - AccessedOpaques, ComputeGoalFastPathOutcome, ExternalRegionConstraints, - FetchEligibleAssocItemResponse, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, - QueryResultOrRerunNonErased, RerunCondition, RerunNonErased, RerunReason, RerunResultExt, - SmallCopyList, + AccessedOpaques, ExternalRegionConstraints, FetchEligibleAssocItemResponse, MaybeInfo, + NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunCondition, + RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, }; use rustc_type_ir::{ self as ty, CanonicalVarValues, ClauseKind, InferCtxtLike, Interner, MayBeErased, @@ -32,6 +31,9 @@ use crate::delegate::SolverDelegate; use crate::normalize::{NormalizationFolder, NormalizationWasAmbiguous}; use crate::placeholder::BoundVarReplacer; use crate::resolve::eager_resolve_vars; +use crate::solve::eval_ctxt::fast_path::{ + RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress, +}; use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ @@ -41,6 +43,7 @@ use crate::solve::{ VisibleForLeakCheck, inspect, }; +mod fast_path; mod probe; mod solver_region_constraints; @@ -90,12 +93,6 @@ impl CurrentGoalKind { } } -#[derive(Debug)] -enum RerunDecision { - Yes, - No, - EagerlyPropagateToParent, -} pub struct EvalCtxt<'a, D, I = ::Interner> where D: SolverDelegate, @@ -227,8 +224,25 @@ where span: I::Span, stalled_on: Option>, ) -> Result, NoSolution> { + // Run fast paths *before* building an `EvalCtxt`, saving a little bit of time. + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); + } + + if let Some(res) = compute_goal_fast_path(self, goal, span) { + return Ok(res); + } + let result = EvalCtxt::enter_root(self, self.cx().recursion_limit(), span, |ecx| { - ecx.evaluate_goal(GoalSource::Misc, goal, stalled_on) + // Fast paths handled above + ecx.evaluate_goal_no_fast_paths(GoalSource::Misc, goal) }); match result { @@ -286,12 +300,6 @@ where } } -#[derive(Debug, Clone, Copy)] -enum RerunStalled { - WontMakeProgress(Certainty), - MayMakeProgress, -} - impl<'a, D, I> EvalCtxt<'a, D> where D: SolverDelegate, @@ -484,125 +492,35 @@ where goal: Goal, stalled_on: Option>, ) -> Result, NoSolutionOrRerunNonErased> { - let (normalization_nested_goals, goal_evaluation) = - self.evaluate_goal_raw(source, goal, stalled_on)?; - assert!(normalization_nested_goals.is_empty()); - Ok(goal_evaluation) - } - - /// This is a fast path optimization: - /// If we have run this goal before, and it was stalled, check that any of the goal's - /// args have changed. This is a cheap way to determine that if we were to rerun this goal now, - /// it will remain stalled since it'll canonicalize the same way and evaluation is pure. - /// Therefore, we can skip this rerun - fn rerunning_stalled_goal_may_make_progress( - &self, - stalled_on: Option<&GoalStalledOn>, - ) -> RerunStalled { - use RerunStalled::*; - - // If fast paths are turned off, then we assume all goals can always make progress - if self.delegate.disable_trait_solver_fast_paths() { - return MayMakeProgress; - } - - // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = - stalled_on - else { - return MayMakeProgress; - }; - - // If any of the stalled goal's generic arguments changed, - // rerunning might make progress so we should rerun. - if stalled_vars.iter().any(|value| self.delegate.is_changed_arg(*value)) { - return MayMakeProgress; - } - - // If some inference took place in any of the sub roots, - // rerunning might make progress so we should rerun. - if sub_roots.iter().any(|&vid| self.delegate.sub_unification_table_root_var(vid) != vid) { - return MayMakeProgress; + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self.delegate, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); } - match reason { - GoalStalledOnReason::FastPath => { - // fastpath is never because of opaques, we can skip this check - } - &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { - // If any opaques changed in the opaque type storage, - // rerunning might make progress so we should rerun. - if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) - { - // Unless this goal previously succeeded in erased mode. - // If the stalled goal successfully evaluated while erasing opaque types, - // and the current state of the opaque type storage is not different in a way that is - // relevant, this stalled goal cannot make any progress and we set this variable to true. - let mut previous_erased_run_is_still_valid = false; - - if let &SucceededInErased::Yes { accessed_opaques } = - previously_succeeded_in_erased - { - match self.should_rerun_after_erased_canonicalization( - accessed_opaques, - self.typing_mode(), - &self.delegate.clone_opaque_types_lookup_table(), - ) { - RerunDecision::Yes => {} - RerunDecision::EagerlyPropagateToParent => { - unreachable!( - "we never retry stalled queries if the parent was erased" - ) - } - RerunDecision::No => { - previous_erased_run_is_still_valid = true; - } - } - } - - if !previous_erased_run_is_still_valid { - return MayMakeProgress; - } - } - } + if let Some(res) = compute_goal_fast_path(self.delegate, goal, self.origin_span) { + return Ok(res); } - // Otherwise, we can be sure that this stalled goal cannot make any progress - // and we can exit early. - WontMakeProgress(stalled_certainty) + self.evaluate_goal_no_fast_paths(source, goal) } - /// This is a fast path optimization: - /// See the docs on [`ComputeGoalFastPathOutcome`] - pub fn compute_goal_fast_path( - &self, + // Outlining and `#[cold]` matter here because fast paths make it less likely to get here. + #[cold] + #[inline(never)] + fn evaluate_goal_no_fast_paths( + &mut self, + source: GoalSource, goal: Goal, - ) -> Option<(NestedNormalizationGoals, GoalEvaluation)> { - if self.delegate.disable_trait_solver_fast_paths() { - return None; - } - - match self.delegate.compute_goal_fast_path(goal, self.origin_span) { - ComputeGoalFastPathOutcome::NoFastPath => None, - ComputeGoalFastPathOutcome::TriviallyHolds => Some(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: Certainty::Yes, - has_changed: HasChanged::No, - stalled_on: None, - }, - )), - ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: Certainty::AMBIGUOUS, - has_changed: HasChanged::No, - stalled_on: Some(stalled_on), - }, - )), - } + ) -> Result, NoSolutionOrRerunNonErased> { + let (normalization_nested_goals, goal_evaluation) = self.evaluate_goal_raw(source, goal)?; + assert!(normalization_nested_goals.is_empty()); + Ok(goal_evaluation) } /// Recursively evaluates `goal`, returning the nested goals in case @@ -616,36 +534,7 @@ where &mut self, source: GoalSource, goal: Goal, - stalled_on: Option>, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let RerunStalled::WontMakeProgress(stalled_certainty) = - self.rerunning_stalled_goal_may_make_progress(stalled_on.as_ref()) - { - return Ok(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: stalled_certainty, - has_changed: HasChanged::No, - stalled_on, - }, - )); - } - - self.evaluate_goal_cold(source, goal) - } - - #[cold] - #[inline(never)] - pub(super) fn evaluate_goal_cold( - &mut self, - source: GoalSource, - goal: Goal, - ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let Some(res) = self.compute_goal_fast_path(goal) { - return Ok(res); - } - // We only care about one entry per `OpaqueTypeKey` here, // so we only canonicalize the lookup table and ignore // duplicate entries. @@ -711,7 +600,7 @@ where &mut inspect::ProofTreeBuilder::new_noop(), ); - let should_rerun = self.should_rerun_after_erased_canonicalization( + let should_rerun = should_rerun_after_erased_canonicalization( accessed_opaques, self.typing_mode(), &opaque_types, @@ -872,100 +761,6 @@ where } } - fn should_rerun_after_erased_canonicalization( - &self, - AccessedOpaques { reason: _, rerun }: AccessedOpaques, - original_typing_mode: TypingMode, - parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], - ) -> RerunDecision { - let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); - let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { - if defids.as_ref().is_empty() { - RerunDecision::No - } else if opaques - .iter() - .chain(parent_opaque_defids) - .any(|opaque| defids.as_ref().contains(&opaque)) - { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - let any_opaque_has_infer_as_hidden = || { - if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - - let res = match (rerun, original_typing_mode) { - // ============================= - (RerunCondition::Never, _) => RerunDecision::No, - // ============================= - (_, TypingMode::ErasedNotCoherence(MayBeErased)) => { - RerunDecision::EagerlyPropagateToParent - } - // ============================= - // In coherence, we never switch to erased mode, so we will never register anything - // in the rerun state, so we should've taken the first branch of this match - (_, TypingMode::Coherence) => unreachable!(), - // ============================= - (RerunCondition::Always, _) => RerunDecision::Yes, - // ============================= - ( - RerunCondition::OpaqueInStorage(..), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::Yes, - ( - RerunCondition::OpaqueInStorage(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - // ============================= - (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { - any_opaque_has_infer_as_hidden() - } - ( - RerunCondition::AnyOpaqueHasInferAsHidden, - TypingMode::PostBorrowck { .. } - | TypingMode::PostAnalysis - | TypingMode::Codegen - | TypingMode::PostTypeckUntilBorrowck { .. }, - ) => RerunDecision::No, - // ============================= - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::No, - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, - ) => { - if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { - RerunDecision::Yes - } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { - RerunDecision::Yes - } else { - RerunDecision::No - } - } - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - }; - - debug!( - "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" - ); - - res - } - pub(super) fn compute_goal( &mut self, goal: Goal, @@ -1797,6 +1592,103 @@ where } } +#[derive(Debug)] +enum RerunDecision { + Yes, + No, + EagerlyPropagateToParent, +} + +fn should_rerun_after_erased_canonicalization( + AccessedOpaques { reason: _, rerun }: AccessedOpaques, + original_typing_mode: TypingMode, + parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], +) -> RerunDecision { + let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); + let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { + if defids.as_ref().is_empty() { + RerunDecision::No + } else if opaques + .iter() + .chain(parent_opaque_defids) + .any(|opaque| defids.as_ref().contains(&opaque)) + { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + let any_opaque_has_infer_as_hidden = || { + if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + + let res = match (rerun, original_typing_mode) { + // ============================= + (RerunCondition::Never, _) => RerunDecision::No, + // ============================= + (_, TypingMode::ErasedNotCoherence(MayBeErased)) => RerunDecision::EagerlyPropagateToParent, + // ============================= + // In coherence, we never switch to erased mode, so we will never register anything + // in the rerun state, so we should've taken the first branch of this match + (_, TypingMode::Coherence) => unreachable!(), + // ============================= + (RerunCondition::Always, _) => RerunDecision::Yes, + // ============================= + (RerunCondition::OpaqueInStorage(..), TypingMode::PostAnalysis | TypingMode::Codegen) => { + RerunDecision::Yes + } + ( + RerunCondition::OpaqueInStorage(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + // ============================= + (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { + any_opaque_has_infer_as_hidden() + } + ( + RerunCondition::AnyOpaqueHasInferAsHidden, + TypingMode::PostBorrowck { .. } + | TypingMode::PostAnalysis + | TypingMode::Codegen + | TypingMode::PostTypeckUntilBorrowck { .. }, + ) => RerunDecision::No, + // ============================= + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), + TypingMode::PostAnalysis | TypingMode::Codegen, + ) => RerunDecision::No, + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, + ) => { + if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { + RerunDecision::Yes + } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { + RerunDecision::Yes + } else { + RerunDecision::No + } + } + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + }; + + debug!( + "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" + ); + + res +} + /// Do not call this directly, use the `tcx` query instead. pub fn evaluate_root_goal_for_proof_tree_raw_provider< D: SolverDelegate, diff --git a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs index e084ae077b561..028af94012484 100644 --- a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs @@ -67,7 +67,7 @@ where let ( NestedNormalizationGoals(nested_goals), GoalEvaluation { goal: _, certainty, stalled_on: _, has_changed: _ }, - ) = self.evaluate_goal_raw(GoalSource::TypeRelating, normalizes_to, None)?; + ) = self.evaluate_goal_raw(GoalSource::TypeRelating, normalizes_to)?; trace!(?nested_goals); From adbd01966efaabdab8f3b4ef7de3d504ccfb2e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 25 Jun 2026 13:26:20 +0200 Subject: [PATCH 8/8] fast path when adding goals --- .../src/solve/eval_ctxt/fast_path.rs | 30 +++++++++++----- .../src/solve/eval_ctxt/mod.rs | 34 ++++++++++++++++--- .../rustc_next_trait_solver/src/solve/mod.rs | 2 +- .../src/solve/fulfill.rs | 18 +++++++++- 4 files changed, 69 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs index 978d97f184797..6a7d49b36b64f 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -5,19 +5,16 @@ //! //! For debugging, fast paths can be disabled using `-Zdisable-fast-paths`. -use rustc_type_ir::InferCtxtLike; -use rustc_type_ir::Interner; use rustc_type_ir::inherent::*; -use rustc_type_ir::solve::Goal; use rustc_type_ir::solve::{ - Certainty, ComputeGoalFastPathOutcome, GoalStalledOn, GoalStalledOnReason, SucceededInErased, + Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnReason, + SucceededInErased, }; +use rustc_type_ir::{InferCtxtLike, Interner}; use crate::delegate::SolverDelegate; -use crate::solve::GoalEvaluation; -use crate::solve::HasChanged; -use crate::solve::eval_ctxt::RerunDecision; -use crate::solve::eval_ctxt::should_rerun_after_erased_canonicalization; +use crate::solve::eval_ctxt::{RerunDecision, should_rerun_after_erased_canonicalization}; +use crate::solve::{GoalEvaluation, HasChanged}; #[derive(Debug, Clone, Copy)] pub(super) enum RerunStalled { @@ -29,6 +26,7 @@ pub(super) enum RerunStalled { /// args have changed. This is a cheap way to determine that if we were to rerun this goal now, /// it will remain stalled since it'll canonicalize the same way and evaluation is pure. /// Therefore, we can skip this rerun +#[inline] pub(super) fn rerunning_stalled_goal_may_make_progress( delegate: &D, stalled_on: Option<&GoalStalledOn>, @@ -106,9 +104,23 @@ where WontMakeProgress(stalled_certainty) } +#[cold] +#[inline(never)] +pub(super) fn compute_goal_fast_path_cold( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + compute_goal_fast_path(delegate, goal, origin_span) +} + /// This is a fast path optimization: /// See the docs on [`ComputeGoalFastPathOutcome`] -pub(super) fn compute_goal_fast_path( +pub fn compute_goal_fast_path( delegate: &D, goal: Goal, origin_span: I::Span, diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index db5634a796611..74eda54007c60 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -34,6 +34,7 @@ use crate::resolve::eager_resolve_vars; use crate::solve::eval_ctxt::fast_path::{ RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress, }; +use crate::solve::fast_path::compute_goal_fast_path_cold; use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ @@ -43,7 +44,7 @@ use crate::solve::{ VisibleForLeakCheck, inspect, }; -mod fast_path; +pub mod fast_path; mod probe; mod solver_region_constraints; @@ -236,7 +237,13 @@ where }); } - if let Some(res) = compute_goal_fast_path(self, goal, span) { + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self, goal, span) + { return Ok(res); } @@ -503,7 +510,13 @@ where }); } - if let Some(res) = compute_goal_fast_path(self.delegate, goal, self.origin_span) { + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self.delegate, goal, self.origin_span) + { return Ok(res); } @@ -901,7 +914,20 @@ where ty::Unnormalized::new_wip(goal.predicate), )?; self.inspect.add_goal(self.delegate, self.max_input_universe, source, goal); - self.nested_goals.push((source, goal, None)); + + if let Some(GoalEvaluation { goal, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(self.delegate, goal, self.origin_span) + { + match certainty { + // We're done here + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.nested_goals.push((source, goal, stalled_on)); + } + } + } else { + self.nested_goals.push((source, goal, None)); + } Ok(()) } diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index c1dad4c7a4aca..27f34e8887a1b 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -29,7 +29,7 @@ use tracing::instrument; pub use self::eval_ctxt::{ EvalCtxt, GenerateProofTree, SolverDelegateEvalExt, - evaluate_root_goal_for_proof_tree_raw_provider, + evaluate_root_goal_for_proof_tree_raw_provider, fast_path, }; use crate::delegate::SolverDelegate; use crate::solve::assembly::Candidate; diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 0944da1f93dfe..4c2c92ebc5072 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -7,6 +7,7 @@ use rustc_infer::traits::{ FromSolverError, PredicateObligation, PredicateObligations, TraitEngine, }; use rustc_middle::ty::{self, TyCtxt, TyVid, TypeVisitableExt, TypingMode}; +use rustc_next_trait_solver::solve::fast_path::compute_goal_fast_path; use rustc_next_trait_solver::solve::{ GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _, StalledOnCoroutines, @@ -184,7 +185,22 @@ where obligation: PredicateObligation<'tcx>, ) { assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots()); - self.obligations.register(obligation, None); + + let delegate = <&SolverDelegate<'tcx>>::from(infcx); + if let Some(GoalEvaluation { goal: _, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(delegate, obligation.as_goal(), obligation.cause.span) + { + // If we can take the fast path, don't even bother adding the goal to obligations, + // or if `Certainty::Maybe`, add it with precise stalled_on information. + match certainty { + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.obligations.register(obligation, stalled_on); + } + } + } else { + self.obligations.register(obligation, None); + } } fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec {