Reject recursive user-defined types at type-check time

source/compiler/qsc_fir_transforms/src/defunctionalize.rs

@@ -36,6 +36,19 @@
 //! - **Diagnostics:** [`Error::ExcessiveSpecializations`] is a non-fatal
 //!   warning. Other errors are fatal because the intermediate FIR may violate
 //!   downstream invariants.
+//! - **Relies on an acyclic UDT graph.** Several type walks in this pass and
+//!   its submodules expand `Ty::Udt` through the referenced type's definition
+//!   and keep descending, with no visited set — `ty_contains_arrow_through_udts`,
+//!   `analysis::extract_arrow_params_from_ty`, `analysis::output_path_resolves_to_arrow`,
+//!   and the `resolve_udt_ty` helpers. They terminate only because a
+//!   user-defined type cannot reference itself. The Q# type checker enforces
+//!   this in `qsc_frontend::typeck::check`, rejecting any cyclic declaration
+//!   with `Qdk.Qsc.TypeCk.RecursiveUdt` before HIR passes run; the guarantee
+//!   covers the package under compilation and extends to the whole store only
+//!   where dependency errors are also gated. Q# has no indirection primitive,
+//!   so `A[]` and `A -> Int` are descended through exactly as a bare `A` is and
+//!   are equally fatal — this pass is where the arrow-mediated case was
+//!   originally observed to overflow the stack.
 //! - Synthesized expressions use `EMPTY_EXEC_RANGE`;
 //!   `crate::exec_graph_rebuild` repairs exec graphs later.
 
@@ -910,6 +923,8 @@ pub(crate) fn ty_contains_arrow(ty: &Ty) -> bool {
 /// callable whose parameter is a UDT containing a callable field keeps the loop
 /// running until that nested callable field is specialized. The rewrite helpers
 /// still use `ty_contains_arrow`, where UDTs intentionally remain opaque.
+///
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 fn ty_contains_arrow_through_udts(store: &PackageStore, ty: &Ty) -> bool {
     match ty {
         Ty::Arrow(_) => true,
@@ -1520,6 +1535,7 @@ pub(super) fn has_multiple_forwarded_callable_arrays(
     static_callable_array_positions(package, group).len() >= 2
 }
 
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 fn resolve_udt_ty(package: &Package, ty: &Ty) -> Ty {
     match ty {
         Ty::Udt(Res::Item(item_id)) => {

source/compiler/qsc_fir_transforms/src/defunctionalize/analysis.rs

@@ -193,6 +193,8 @@ struct ArrowParamExtraction<'a> {
 ///
 /// UDTs are expanded to their pure type so callable fields inside nested
 /// newtypes are treated the same way as tuple fields.
+///
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 fn extract_arrow_params_from_ty(
     context: &ArrowParamExtraction<'_>,
     param_ty: &Ty,
@@ -1452,6 +1454,8 @@ fn resolve_callee_projection(
 
 /// Reports whether following `path` into the (possibly nested tuple) type `ty`
 /// lands on an arrow type.
+///
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 fn output_path_resolves_to_arrow(store: &PackageStore, ty: &Ty, path: &[usize]) -> bool {
     match ty {
         Ty::Arrow(_) => path.is_empty(),

source/compiler/qsc_fir_transforms/src/defunctionalize/rewrite.rs

@@ -5049,6 +5049,8 @@ fn local_ty_contains_arrow_through_udts(package: &Package, ty: &Ty) -> bool {
 /// Resolves a type through user-defined-type wrappers to its underlying
 /// structural type, recursing into tuples, arrays, and arrow inputs and
 /// outputs.
+///
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 fn resolve_udt_ty(package: &Package, ty: &Ty) -> Ty {
     match ty {
         Ty::Udt(Res::Item(item_id)) => {

source/compiler/qsc_fir_transforms/src/defunctionalize/specialize.rs

@@ -1921,6 +1921,8 @@ fn callable_input_contains_arrow(package: &Package, callable: LocalItemId) -> bo
 /// A residual `Ty::Udt` (one [`resolve_udt_ty`] could not expand, e.g. a
 /// foreign or non-`Ty` item) conservatively counts as containing an arrow so an
 /// unknown shape is never misclassified as arrow-free.
+///
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 fn ty_contains_arrow_through_udts(package: &Package, ty: &Ty) -> bool {
     match resolve_udt_ty(package, ty) {
         Ty::Arrow(_) | Ty::Udt(_) => true,
@@ -5341,6 +5343,8 @@ fn remove_ty_at_nested_path(package: &Package, ty: &Ty, path: &[usize]) -> Ty {
 /// structural view that analysis used so a path like `cfg::Inner::Op` can remove the arrow
 /// field from the specialized callable's input type. Non-UDT leaves are preserved, and nested
 /// tuples, arrays, and arrows are rebuilt with any UDTs inside them expanded as well.
+///
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 fn resolve_udt_ty(package: &Package, ty: &Ty) -> Ty {
     match ty {
         Ty::Udt(Res::Item(item_id)) => {

source/compiler/qsc_fir_transforms/src/return_unify/slot.rs

@@ -596,6 +596,8 @@ fn create_default_value_kind(
 }
 
 /// Read-only check whether `ty` has a synthesizable classical default.
+///
+/// Unguarded UDT recursion; terminates only because the frontend rejects cyclic UDTs.
 pub(super) fn is_type_defaultable(package: &Package, package_id: PackageId, ty: &Ty) -> bool {
     match ty {
         Ty::Prim(

source/compiler/qsc_fir_transforms/src/return_unify/tests/contracts_and_errors.rs

@@ -9,6 +9,7 @@ use qsc_fir::{
 use rustc_hash::FxHashMap;
 
 use crate::fir_builder::alloc_expr_stmt;
+use crate::test_utils::frontend_error_codes;
 
 use super::*;
 
@@ -611,10 +612,10 @@ fn defaultable_type_with_early_return_succeeds() {
 
 #[test]
 fn recursive_udt_early_return_fails_before_return_unify() {
-    // Recursive UDTs (e.g. `newtype Tree = (Int, Tree[])`) are definable
-    // in Q# but produce a compile error at the frontend before reaching
-    // return_unify. This documents that L7 (recursive-UDT defaultability)
-    // is covered by language-level rejection.
+    // A recursive UDT is rejected by the Q# type checker, so it never reaches this crate.
+    // That is what lets `return_unify` and the other transforms expand UDT definitions
+    // structurally without a visited set. The diagnostic itself is pinned by the
+    // `recursive_udt_*` tests in `qsc_frontend::typeck::tests`.
     let source = indoc! {r#"
         namespace Test {
             newtype Tree = (Data : Int, Children : Tree[]);
@@ -630,23 +631,10 @@ fn recursive_udt_early_return_fails_before_return_unify() {
         }
     "#};
 
-    let (_store, _pkg_id, result) =
-        compile_and_run_pipeline_to_with_errors(source, PipelineStage::ReturnUnify);
-
-    // The program should either fail at the frontend (cyclic UDT) or
-    // succeed if the frontend resolves it. Either way, it should not
-    // panic in return_unify.
-    // If errors exist, they should not be return_unify panics.
-    for err in &result.errors {
-        if let crate::PipelineError::ReturnUnify(ru_err) = err {
-            // Any return_unify error is acceptable (diagnostic, not panic).
-            // We just verify it didn't panic.
-            assert!(
-                !format!("{ru_err:?}").contains("panic"),
-                "return_unify should not panic on recursive UDT: {ru_err:?}"
-            );
-        }
-    }
+    assert_eq!(
+        frontend_error_codes(source),
+        vec!["Qdk.Qsc.TypeCk.RecursiveUdt".to_string()]
+    );
 }
 
 #[test]

source/compiler/qsc_fir_transforms/src/test_utils.rs

@@ -261,6 +261,31 @@ pub fn compile_to_fir(source: &str) -> (fir::PackageStore, fir::PackageId) {
     compile_to_fir_with_capabilities(source, TargetCapabilityFlags::empty())
 }
 
+/// Compiles Q# source and returns the frontend diagnostic codes instead of
+/// asserting that compilation succeeded.
+///
+/// Use this to pin source that the frontend is expected to reject, which the
+/// asserting `compile_to_fir*` helpers cannot express.
+#[cfg(test)]
+pub(crate) fn frontend_error_codes(source: &str) -> Vec<String> {
+    use miette::Diagnostic;
+
+    with_cached_stdlib_store(TargetCapabilityFlags::empty(), |store, std_id| {
+        let sources = SourceMap::new(vec![("test.qs".into(), source.into())], None);
+        let unit = frontend_compile::compile(
+            store,
+            &[(PackageId::CORE, None), (std_id, None)],
+            sources,
+            TargetCapabilityFlags::empty(),
+            LanguageFeatures::default(),
+        );
+        unit.errors
+            .iter()
+            .filter_map(|error| error.code().map(|code| code.to_string()))
+            .collect()
+    })
+}
+
 /// Compiles Q# source through core+std → HIR passes → FIR lowering using the
 /// given target capabilities.
 ///

source/compiler/qsc_fir_transforms/src/udt_erase.rs

@@ -27,6 +27,17 @@
 //!   callable bodies in place; the pipeline driver unconditionally rebuilds the
 //!   exec graph of every reachable spec in every reachable package afterwards,
 //!   so this pass no longer tracks or returns which specs it mutated.
+//! - **Relies on an acyclic UDT graph.** `resolve_ty` recurses through `Udt`,
+//!   `Array`, `Tuple`, and `Arrow` with no visited set, which is sound only
+//!   because a user-defined type cannot reference itself. The Q# type checker
+//!   enforces this in `qsc_frontend::typeck::check`, which rejects any cyclic
+//!   declaration with `Qdk.Qsc.TypeCk.RecursiveUdt` before HIR passes run. The
+//!   guarantee covers the package under compilation and extends to the whole
+//!   store only where dependency errors are also gated. Note that no form of
+//!   indirection exempts a cycle: `A[]` and `A -> Int` are erased structurally
+//!   just as a bare `A` is, so a cyclic type has no finite erased form at all —
+//!   a visited-set guard would terminate into a state that still violates
+//!   `PostUdtErase` rather than into a correct one.
 //! - Synthesized expressions use `EMPTY_EXEC_RANGE`;
 //!   [`crate::exec_graph_rebuild`] rebuilds exec graphs later.
 

source/compiler/qsc_frontend/src/incremental/tests.rs

@@ -348,6 +348,40 @@ fn errors_across_multiple_lines() {
     .assert_debug_eq(&labels);
 }
 
+#[test]
+fn recursive_udt_reported_once_across_fragments() {
+    let store = PackageStore::new(compile::core());
+    let mut compiler = Compiler::new(
+        &store,
+        &Vec::new(),
+        TargetCapabilityFlags::all(),
+        LanguageFeatures::default(),
+    );
+    let mut unit = CompileUnit::new(store.peek_package_id());
+
+    let errors = compiler
+        .compile_fragments(
+            &mut unit,
+            "line_1",
+            "struct Foo { Bar : Foo }",
+            fail_on_error,
+        )
+        .expect_err("should fail");
+    assert_eq!(
+        errors
+            .iter()
+            .filter_map(|e| e.code().map(|c| c.to_string()))
+            .collect::<Vec<_>>(),
+        vec!["Qdk.Qsc.TypeCk.RecursiveUdt".to_string()]
+    );
+
+    // The checker is long-lived across fragments, so the already-reported type must not be
+    // re-reported when a later fragment is compiled.
+    compiler
+        .compile_fragments(&mut unit, "line_2", "let x = 1;", fail_on_error)
+        .expect("should succeed");
+}
+
 #[test]
 fn continue_after_parse_error() {
     let store = PackageStore::new(compile::core());

source/compiler/qsc_frontend/src/typeck.rs

@@ -240,6 +240,16 @@ enum ErrorKind {
         span: Span,
         name: String,
     },
+    #[error("user-defined type `{name}` is recursive")]
+    #[help(
+        "a user-defined type cannot contain itself, directly or through other types; arrays and callable types do not break the cycle, so `{name}[]` and `{name} -> _` are recursive as well"
+    )]
+    #[diagnostic(code("Qdk.Qsc.TypeCk.RecursiveUdt"))]
+    RecursiveUdt {
+        name: String,
+        #[label]
+        span: Span,
+    },
     #[error("expected {expected} parameters for constraint, found {found}")]
     #[diagnostic(code("Qdk.Qsc.TypeCk.IncorrectNumberOfConstraintParameters"))]
     IncorrectNumberOfConstraintParameters {