Avoid mono collection hangs from recursive type growth

compiler/rustc_middle/src/ty/instance.rs

@@ -343,7 +343,7 @@ impl<'tcx> InstanceKind<'tcx> {
     }
 }
 
-fn type_length<'tcx>(item: impl TypeVisitable<TyCtxt<'tcx>>) -> usize {
+pub fn type_length<'tcx>(item: impl TypeVisitable<TyCtxt<'tcx>>) -> usize {
     struct Visitor<'tcx> {
         type_length: usize,
         cache: FxHashMap<Ty<'tcx>, usize>,

compiler/rustc_middle/src/ty/mod.rs

@@ -86,7 +86,7 @@ pub use self::context::{
     CtxtInterners, CurrentGcx, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt, TyCtxtFeed, tls,
 };
 pub use self::fold::*;
-pub use self::instance::{Instance, InstanceKind, ReifyReason};
+pub use self::instance::{Instance, InstanceKind, ReifyReason, type_length};
 pub(crate) use self::list::RawList;
 pub use self::list::{List, ListWithCachedTypeInfo};
 pub use self::opaque_types::OpaqueTypeKey;

compiler/rustc_monomorphize/src/collector.rs

@@ -227,7 +227,7 @@ use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCoercion};
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::{
     self, GenericArgs, GenericParamDefKind, Instance, InstanceKind, Ty, TyCtxt, TypeFoldable,
-    TypeVisitable, TypeVisitableExt, TypeVisitor, Unnormalized, VtblEntry,
+    TypeVisitable, TypeVisitableExt, TypeVisitor, Unnormalized, VtblEntry, type_length,
 };
 use rustc_middle::util::Providers;
 use rustc_middle::{bug, span_bug};
@@ -356,6 +356,7 @@ fn collect_items_root<'tcx>(
     starting_item: Spanned<MonoItem<'tcx>>,
     state: &SharedState<'tcx>,
     recursion_limit: Limit,
+    type_length_limit: Limit,
 ) {
     if !state.visited.lock().insert(starting_item.node) {
         // We've been here already, no need to search again.
@@ -369,6 +370,7 @@ fn collect_items_root<'tcx>(
         &mut recursion_depths,
         recursion_limit,
         CollectionMode::UsedItems,
+        type_length_limit,
     );
 }
 
@@ -377,14 +379,18 @@ fn collect_items_root<'tcx>(
 ///
 /// `mode` determined whether we are scanning for [used items][CollectionMode::UsedItems]
 /// or [mentioned items][CollectionMode::MentionedItems].
-#[instrument(skip(tcx, state, recursion_depths, recursion_limit), level = "debug")]
+#[instrument(
+    skip(tcx, state, recursion_depths, recursion_limit, type_length_limit),
+    level = "debug"
+)]
 fn collect_items_rec<'tcx>(
     tcx: TyCtxt<'tcx>,
     starting_item: Spanned<MonoItem<'tcx>>,
     state: &SharedState<'tcx>,
     recursion_depths: &mut DefIdMap<usize>,
     recursion_limit: Limit,
     mode: CollectionMode,
+    type_length_limit: Limit,
 ) {
     let mut used_items = MonoItems::new();
     let mut mentioned_items = MonoItems::new();
@@ -462,13 +468,14 @@ fn collect_items_rec<'tcx>(
             // Sanity check whether this ended up being collected accidentally
             debug_assert!(tcx.should_codegen_locally(instance));
 
-            // Keep track of the monomorphization recursion depth
+            // Check for recursive monomorphization before collecting uses
             recursion_depth_reset = Some(check_recursion_limit(
                 tcx,
                 instance,
                 starting_item.span,
                 recursion_depths,
                 recursion_limit,
+                type_length_limit,
             ));
 
             rustc_data_structures::stack::ensure_sufficient_stack(|| {
@@ -594,6 +601,7 @@ fn collect_items_rec<'tcx>(
                 recursion_depths,
                 recursion_limit,
                 CollectionMode::UsedItems,
+                type_length_limit,
             );
         }
     }
@@ -608,6 +616,7 @@ fn collect_items_rec<'tcx>(
             recursion_depths,
             recursion_limit,
             CollectionMode::MentionedItems,
+            type_length_limit,
         );
     }
 
@@ -651,6 +660,7 @@ fn check_recursion_limit<'tcx>(
     span: Span,
     recursion_depths: &mut DefIdMap<usize>,
     recursion_limit: Limit,
+    type_length_limit: Limit,
 ) -> (DefId, usize) {
     let def_id = instance.def_id();
     let recursion_depth = recursion_depths.get(&def_id).cloned().unwrap_or(0);
@@ -664,10 +674,20 @@ fn check_recursion_limit<'tcx>(
         recursion_depth
     };
 
-    // Code that needs to instantiate the same function recursively
-    // more than the recursion limit is assumed to be causing an
-    // infinite expansion.
-    if !recursion_limit.value_within_limit(adjusted_recursion_depth) {
+    // Recursive monomorphization can grow instance args exponentially with polynomial
+    // recursion depth. Start checking type lengths around `ilog2(type_length_limit.0)`
+    // to avoid hanging on enormous instance arguments.
+    let type_length_check_depth = type_length_limit.0.checked_ilog2().unwrap_or(0).max(4) as usize;
+    let recursive_type_growth_limit_reached = recursion_depth >= type_length_check_depth
+        && !type_length_limit.value_within_limit(type_length(instance.args));
+
+    // Code that needs to instantiate the same function recursively more
+    // than the recursion limit is assumed to be causing an infinite
+    // expansion. Bail out earlier if recursive instantiations have already
+    // produced instance args exceeding the type length limit.
+    if !recursion_limit.value_within_limit(adjusted_recursion_depth)
+        || recursive_type_growth_limit_reached
+    {
         let def_span = tcx.def_span(def_id);
         let def_path_str = tcx.def_path_str(def_id);
         tcx.dcx().emit_fatal(RecursionLimit { span, instance, def_span, def_path_str });
@@ -1827,9 +1847,17 @@ pub(crate) fn collect_crate_mono_items<'tcx>(
     };
     let recursion_limit = tcx.recursion_limit();
 
+    let type_length_limit = tcx.type_length_limit();
+
     tcx.sess.time("monomorphization_collector_graph_walk", || {
         par_for_each_in(roots, |root| {
-            collect_items_root(tcx, dummy_spanned(*root), &state, recursion_limit);
+            collect_items_root(
+                tcx,
+                dummy_spanned(*root),
+                &state,
+                recursion_limit,
+                type_length_limit,
+            );
         });
     });
 

tests/ui/codegen/overflow-during-mono.rs

@@ -1,4 +1,3 @@
-//~ ERROR overflow evaluating the requirement `for<'a> {closure@$DIR/overflow-during-mono.rs:14:41: 14:44}: FnMut(&'a _)`
 //@ build-fail
 //@ compile-flags: -Zwrite-long-types-to-disk=yes
 
@@ -15,6 +14,7 @@ fn quicksort<It: Clone + Iterator<Item = T>, I: IntoIterator<IntoIter = It>, T:
             let greater = i.filter(|y| &x <= y);
 
             let mut v = quicksort(less);
+            //~^ ERROR reached the recursion limit while instantiating
             let u = quicksort(greater);
             v.push(x);
             v.extend(u);

tests/ui/codegen/overflow-during-mono.stderr

@@ -1,13 +1,18 @@
-error[E0275]: overflow evaluating the requirement `for<'a> {closure@$DIR/overflow-during-mono.rs:14:41: 14:44}: FnMut(&'a _)`
+error: reached the recursion limit while instantiating `quicksort::<Filter<Filter<Filter<..., ...>, ...>, ...>, ..., i32>`
+  --> $DIR/overflow-during-mono.rs:16:25
    |
-   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "64"]` attribute to your crate (`overflow_during_mono`)
-   = note: required for `Filter<IntoIter<i32, 11>, {closure@overflow-during-mono.rs:14:41}>` to implement `Iterator`
-   = note: 31 redundant requirements hidden
-   = note: required for `Filter<Filter<Filter<Filter<Filter<..., ...>, ...>, ...>, ...>, ...>` to implement `Iterator`
-   = note: required for `Filter<Filter<Filter<Filter<Filter<..., ...>, ...>, ...>, ...>, ...>` to implement `IntoIterator`
+LL |             let mut v = quicksort(less);
+   |                         ^^^^^^^^^^^^^^^
+   |
+note: `quicksort` defined here
+  --> $DIR/overflow-during-mono.rs:6:1
+   |
+LL | / fn quicksort<It: Clone + Iterator<Item = T>, I: IntoIterator<IntoIter = It>, T: Ord>(
+LL | |     i: I,
+LL | | ) -> Vec<T> {
+   | |___________^
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/overflow-during-mono.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 
 error: aborting due to 1 previous error
 
-For more information about this error, try `rustc --explain E0275`.

tests/ui/issues/issue-22638.rs

@@ -1,13 +1,12 @@
 //@ build-fail
 
 #![allow(unused)]
-
 #![recursion_limit = "20"]
 #![type_length_limit = "20000000"]
 #![crate_type = "rlib"]
 
 #[derive(Clone)]
-struct A (B);
+struct A(B);
 
 impl A {
     pub fn matches<F: Fn()>(&self, f: &F) {
@@ -25,28 +24,24 @@ enum B {
 impl B {
     pub fn matches<F: Fn()>(&self, f: &F) {
         match self {
-            &B::Variant2(ref factor) => {
-                factor.matches(&|| ())
-            }
-            _ => unreachable!("")
+            &B::Variant2(ref factor) => factor.matches(&|| ()),
+            _ => unreachable!(""),
         }
     }
 }
 
 #[derive(Clone)]
-struct C (D);
+struct C(D);
 
 impl C {
     pub fn matches<F: Fn()>(&self, f: &F) {
         let &C(ref base) = self;
-        base.matches(&|| {
-            C(base.clone()).matches(f)
-        })
+        base.matches(&|| C(base.clone()).matches(f))
     }
 }
 
 #[derive(Clone)]
-struct D (Box<A>);
+struct D(Box<A>);
 
 impl D {
     pub fn matches<F: Fn()>(&self, f: &F) {

tests/ui/issues/issue-22638.stderr

@@ -1,11 +1,11 @@
-error: reached the recursion limit while instantiating `A::matches::<{closure@$DIR/issue-22638.rs:42:23: 42:25}>`
-  --> $DIR/issue-22638.rs:54:9
+error: reached the recursion limit while instantiating `A::matches::<{closure@$DIR/issue-22638.rs:39:23: 39:25}>`
+  --> $DIR/issue-22638.rs:49:9
    |
 LL |         a.matches(f)
    |         ^^^^^^^^^^^^
    |
 note: `A::matches` defined here
-  --> $DIR/issue-22638.rs:13:5
+  --> $DIR/issue-22638.rs:12:5
    |
 LL |     pub fn matches<F: Fn()>(&self, f: &F) {
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

tests/ui/recursion/issue-83150.rs

@@ -1,4 +1,3 @@
-//~ ERROR overflow evaluating the requirement `Map<&mut std::ops::Range<u8>, {closure@$DIR/issue-83150.rs:12:24: 12:27}>: Iterator`
 //@ build-fail
 //@ compile-flags: -Copt-level=0 -Zwrite-long-types-to-disk=yes
 
@@ -10,4 +9,5 @@ fn main() {
 fn func<T: Iterator<Item = u8>>(iter: &mut T) {
     //~^ WARN function cannot return without recursing
     func(&mut iter.map(|x| x + 1))
+    //~^ ERROR reached the recursion limit while instantiating
 }

tests/ui/recursion/issue-83150.stderr

@@ -1,5 +1,5 @@
 warning: function cannot return without recursing
-  --> $DIR/issue-83150.rs:10:1
+  --> $DIR/issue-83150.rs:9:1
    |
 LL | fn func<T: Iterator<Item = u8>>(iter: &mut T) {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot return without recursing
@@ -10,15 +10,19 @@ LL |     func(&mut iter.map(|x| x + 1))
    = help: a `loop` may express intention better if this is on purpose
    = note: `#[warn(unconditional_recursion)]` on by default
 
-error[E0275]: overflow evaluating the requirement `Map<&mut std::ops::Range<u8>, {closure@$DIR/issue-83150.rs:12:24: 12:27}>: Iterator`
+error: reached the recursion limit while instantiating `func::<Map<&mut Map<&mut Map<&mut Map<..., ...>, ...>, ...>, ...>>`
+  --> $DIR/issue-83150.rs:11:5
    |
-   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`issue_83150`)
-   = note: required for `&mut Map<&mut Range<u8>, {closure@issue-83150.rs:12:24}>` to implement `Iterator`
-   = note: 65 redundant requirements hidden
-   = note: required for `&mut Map<&mut Map<&mut Map<&mut Map<&mut ..., ...>, ...>, ...>, ...>` to implement `Iterator`
+LL |     func(&mut iter.map(|x| x + 1))
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: `func` defined here
+  --> $DIR/issue-83150.rs:9:1
+   |
+LL | fn func<T: Iterator<Item = u8>>(iter: &mut T) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/issue-83150.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 
 error: aborting due to 1 previous error; 1 warning emitted
 
-For more information about this error, try `rustc --explain E0275`.

tests/ui/recursion/recursive-mono-type-growth-issue-156272.rs

@@ -0,0 +1,22 @@
+//@ build-fail
+//@ compile-flags: -Zwrite-long-types-to-disk=yes
+
+#![allow(dead_code)]
+
+enum Foo<A> {
+    Fst,
+    Snd(Box<dyn Fn() -> Foo<(A, A)>>),
+}
+
+fn recursive<A>(x: Foo<A>) {
+    match x {
+        Foo::Fst => (),
+        Foo::Snd(f) => recursive(f()),
+        //~^ ERROR reached the recursion limit while instantiating
+    }
+}
+
+fn main() {
+    let p0: Foo<()> = Foo::Fst;
+    recursive(p0);
+}

tests/ui/recursion/recursive-mono-type-growth-issue-156272.stderr

@@ -0,0 +1,16 @@
+error: reached the recursion limit while instantiating `recursive::<(((((((..., ...), ...), ...), ...), ...), ...), ...)>`
+  --> $DIR/recursive-mono-type-growth-issue-156272.rs:14:24
+   |
+LL |         Foo::Snd(f) => recursive(f()),
+   |                        ^^^^^^^^^^^^^^
+   |
+note: `recursive` defined here
+  --> $DIR/recursive-mono-type-growth-issue-156272.rs:11:1
+   |
+LL | fn recursive<A>(x: Foo<A>) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   = note: the full name for the type has been written to '$TEST_BUILD_DIR/recursive-mono-type-growth-issue-156272.long-type-$LONG_TYPE_HASH.txt'
+   = note: consider using `--verbose` to print the full type name to the console
+
+error: aborting due to 1 previous error
+

tests/ui/traits/issue-91949-hangs-on-recursion.rs

@@ -1,4 +1,3 @@
-//~ ERROR overflow evaluating the requirement `<std::iter::Empty<()> as Iterator>::Item == ()`
 //@ build-fail
 //@ compile-flags: -Zinline-mir=no -Zwrite-long-types-to-disk=yes
 
@@ -24,6 +23,7 @@ where
     T: Iterator<Item = ()>,
 {
     recurse(IteratorOfWrapped(elements).map(|t| t.0))
+    //~^ ERROR reached the recursion limit while instantiating
 }
 
 fn main() {

tests/ui/traits/issue-91949-hangs-on-recursion.stderr

@@ -1,5 +1,5 @@
 warning: function cannot return without recursing
-  --> $DIR/issue-91949-hangs-on-recursion.rs:21:1
+  --> $DIR/issue-91949-hangs-on-recursion.rs:20:1
    |
 LL | / fn recurse<T>(elements: T) -> Vec<char>
 LL | |
@@ -13,21 +13,22 @@ LL |       recurse(IteratorOfWrapped(elements).map(|t| t.0))
    = help: a `loop` may express intention better if this is on purpose
    = note: `#[warn(unconditional_recursion)]` on by default
 
-error[E0275]: overflow evaluating the requirement `<std::iter::Empty<()> as Iterator>::Item == ()`
+error: reached the recursion limit while instantiating `recurse::<Map<IteratorOfWrapped<(), Map<..., ...>>, ...>>`
+  --> $DIR/issue-91949-hangs-on-recursion.rs:25:5
    |
-   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "512"]` attribute to your crate (`issue_91949_hangs_on_recursion`)
-note: required for `IteratorOfWrapped<(), std::iter::Empty<()>>` to implement `Iterator`
-  --> $DIR/issue-91949-hangs-on-recursion.rs:14:32
+LL |     recurse(IteratorOfWrapped(elements).map(|t| t.0))
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-LL | impl<T, I: Iterator<Item = T>> Iterator for IteratorOfWrapped<T, I> {
-   |                     --------   ^^^^^^^^     ^^^^^^^^^^^^^^^^^^^^^^^
-   |                     |
-   |                     unsatisfied trait bound introduced here
-   = note: 256 redundant requirements hidden
-   = note: required for `IteratorOfWrapped<(), Map<IteratorOfWrapped<(), Map<..., ...>>, ...>>` to implement `Iterator`
+note: `recurse` defined here
+  --> $DIR/issue-91949-hangs-on-recursion.rs:20:1
+   |
+LL | / fn recurse<T>(elements: T) -> Vec<char>
+LL | |
+LL | | where
+LL | |     T: Iterator<Item = ()>,
+   | |___________________________^
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/issue-91949-hangs-on-recursion.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 
 error: aborting due to 1 previous error; 1 warning emitted
 
-For more information about this error, try `rustc --explain E0275`.