x64: report an error instead of panicking on stack frames larger than i32::MAX

cranelift/codegen/src/isa/aarch64/mod.rs

@@ -74,7 +74,7 @@ impl TargetIsa for AArch64Backend {
     ) -> CodegenResult<CompiledCodeStencil> {
         let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;
 
-        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
+        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane)?;
         let value_labels_ranges = emit_result.value_labels_ranges;
         let buffer = emit_result.buffer;
 

cranelift/codegen/src/isa/pulley_shared/mod.rs

@@ -179,7 +179,7 @@ where
 
         let want_disasm =
             want_disasm || (cfg!(feature = "trace-log") && log::log_enabled!(log::Level::Debug));
-        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
+        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane)?;
         let value_labels_ranges = emit_result.value_labels_ranges;
         let buffer = emit_result.buffer;
 

cranelift/codegen/src/isa/riscv64/mod.rs

@@ -76,7 +76,7 @@ impl TargetIsa for Riscv64Backend {
         let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;
 
         let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug);
-        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
+        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane)?;
         let value_labels_ranges = emit_result.value_labels_ranges;
         let buffer = emit_result.buffer;
 

cranelift/codegen/src/isa/s390x/mod.rs

@@ -73,7 +73,7 @@ impl TargetIsa for S390xBackend {
         let flags = self.flags();
         let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;
 
-        let emit_result = vcode.emit(&regalloc_result, want_disasm, flags, ctrl_plane);
+        let emit_result = vcode.emit(&regalloc_result, want_disasm, flags, ctrl_plane)?;
         let value_labels_ranges = emit_result.value_labels_ranges;
         let buffer = emit_result.buffer;
 

cranelift/codegen/src/isa/x64/abi.rs

@@ -809,6 +809,26 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         insts
     }
 
+    fn validate_frame_layout(frame_layout: &FrameLayout, guard_size: u32) -> CodegenResult<()> {
+        let rsp_relative: u64 =
+            frame_layout.active_size() as u64 + frame_layout.setup_area_size as u64;
+        let rbp_relative: u64 =
+            frame_layout.tail_args_size as u64 + frame_layout.setup_area_size as u64;
+
+        // The stack-probe loop rounds the frame size up to the next guard-page
+        // boundary (align_to in emit.rs) before encoding it as a 32-bit immediate.
+        // A frame size just under the limit can be pushed over it by this rounding,
+        // so reserve a full guard-page margin.
+        let probe_margin = guard_size as u64;
+
+        if rsp_relative.saturating_add(probe_margin) > i32::MAX as u64
+            || rbp_relative > i32::MAX as u64
+        {
+            return Err(crate::CodegenError::ImplLimitExceeded);
+        }
+        Ok(())
+    }
+
     fn gen_memcpy<F: FnMut(Type) -> Writable<Reg>>(
         call_conv: isa::CallConv,
         dst: Reg,
@@ -1343,3 +1363,59 @@ const fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv {
 
     env
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn make_frame_layout(
+        tail_args_size: u32,
+        setup_area_size: u32,
+        clobber_size: u32,
+        fixed_frame_storage_size: u32,
+        outgoing_args_size: u32,
+    ) -> FrameLayout {
+        FrameLayout {
+            word_bytes: 8,
+            incoming_args_size: tail_args_size,
+            tail_args_size,
+            setup_area_size,
+            clobber_size,
+            fixed_frame_storage_size,
+            stackslots_size: 0,
+            outgoing_args_size,
+            clobbered_callee_saves: Vec::new(),
+            function_calls: FunctionCalls::None,
+        }
+    }
+
+    const GUARD_SIZE: u32 = 4096;
+
+    #[test]
+    fn validate_frame_layout_accepts_normal_frame() {
+        let layout = make_frame_layout(64, 16, 64, 1024, 32);
+        assert!(X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE).is_ok());
+    }
+
+    #[test]
+    fn validate_frame_layout_rejects_oversized_fixed_storage() {
+        let layout = make_frame_layout(64, 16, 64, 2_147_483_647, 32);
+        let result = X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn validate_frame_layout_rejects_frame_pushed_over_by_guard_margin() {
+        let just_under = (i32::MAX as u32) - 2000;
+        let layout = make_frame_layout(64, 16, 64, just_under, 0);
+        let result = X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn validate_frame_layout_rejects_oversized_tail_args() {
+        let layout = make_frame_layout(2_147_483_647, 16, 0, 0, 0);
+        let result = X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE);
+        assert!(result.is_err());
+    }
+}

cranelift/codegen/src/isa/x64/mod.rs

@@ -82,7 +82,7 @@ impl TargetIsa for X64Backend {
     ) -> CodegenResult<CompiledCodeStencil> {
         let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?;
 
-        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane);
+        let emit_result = vcode.emit(&regalloc_result, want_disasm, &self.flags, ctrl_plane)?;
         let value_labels_ranges = emit_result.value_labels_ranges;
         let buffer = emit_result.buffer;
 

cranelift/codegen/src/machinst/abi.rs

@@ -555,6 +555,13 @@ pub trait ABIMachineSpec {
         frame_layout: &FrameLayout,
     ) -> SmallVec<[Self::I; 16]>;
 
+    /// Validate that the computed frame layout is supported by this backend.
+    /// Default: always succeeds. Backends with encoding limits (e.g. 32-bit
+    /// immediates) should override this to reject layouts they cannot emit.
+    fn validate_frame_layout(_frame_layout: &FrameLayout, _guard_size: u32) -> CodegenResult<()> {
+        Ok(())
+    }
+
     /// Generate a memcpy invocation. Used to set up struct
     /// args. Takes `src`, `dst` as read-only inputs and passes a temporary
     /// allocator.
@@ -2204,12 +2211,12 @@ impl<M: ABIMachineSpec> Callee<M> {
         spillslots: usize,
         clobbered: Vec<Writable<RealReg>>,
         function_calls: FunctionCalls,
-    ) {
+    ) -> CodegenResult<()> {
         let bytes = M::word_bytes();
         let total_stacksize = self.stackslots_size + bytes * spillslots as u32;
         let mask = M::stack_align(self.call_conv) - 1;
-        let total_stacksize = (total_stacksize + mask) & !mask; // 16-align the stack.
-        self.frame_layout = Some(M::compute_frame_layout(
+        let total_stacksize = (total_stacksize + mask) & !mask;
+        let frame_layout = M::compute_frame_layout(
             self.call_conv,
             &self.flags,
             self.signature(),
@@ -2220,7 +2227,23 @@ impl<M: ABIMachineSpec> Callee<M> {
             self.stackslots_size,
             total_stacksize,
             self.outgoing_args_size,
-        ));
+        );
+
+        // FrameLayout's fields are all u32, so any backend can in principle
+        // overflow a 32-bit sum here; this generic check applies to all targets.
+        // x64 needs it's own specific check.
+        let total: u64 = frame_layout.incoming_args_size as u64
+            + frame_layout.tail_args_size as u64
+            + frame_layout.setup_area_size as u64
+            + frame_layout.clobber_size as u64
+            + frame_layout.fixed_frame_storage_size as u64
+            + frame_layout.outgoing_args_size as u64;
+        if total > u32::MAX as u64 {
+            return Err(CodegenError::ImplLimitExceeded);
+        }
+
+        self.frame_layout = Some(frame_layout);
+        Ok(())
     }
 
     /// Generate a prologue, post-regalloc.

cranelift/codegen/src/machinst/vcode.rs

@@ -739,7 +739,7 @@ impl<I: VCodeInst> VCode<I> {
         want_disasm: bool,
         flags: &settings::Flags,
         ctrl_plane: &mut ControlPlane,
-    ) -> EmitResult
+    ) -> CodegenResult<EmitResult>
     where
         I: VCodeInst,
     {
@@ -783,7 +783,10 @@ impl<I: VCodeInst> VCode<I> {
             regalloc.num_spillslots,
             clobbers,
             function_calls,
-        );
+        )?;
+
+        let guard_size = 1u32 << flags.probestack_size_log2();
+        I::ABIMachineSpec::validate_frame_layout(self.abi.frame_layout(), guard_size)?;
 
         // Emit blocks.
         let mut cur_srcloc = None;
@@ -1164,13 +1167,13 @@ impl<I: VCodeInst> VCode<I> {
         // Store metadata about frame layout in the MachBuffer.
         buffer.set_frame_layout(self.abi.frame_slot_metadata());
 
-        EmitResult {
+        Ok(EmitResult {
             buffer: buffer.finish(&self.constants, ctrl_plane),
             bb_offsets,
             bb_edges,
             disasm: if want_disasm { Some(disasm) } else { None },
             value_labels_ranges,
-        }
+        })
     }
 
     fn monotonize_inst_offsets(&self, inst_offsets: &mut [CodeOffset], func_body_len: u32) {