From 92b0111cac54b797c075303a453f0f97428f2cda Mon Sep 17 00:00:00 2001 From: Soumik15630m Date: Wed, 1 Jul 2026 07:34:14 +0530 Subject: [PATCH 1/2] x64: report an error instead of panicking on oversized stack frames --- cranelift/codegen/src/isa/aarch64/mod.rs | 2 +- .../codegen/src/isa/pulley_shared/mod.rs | 2 +- cranelift/codegen/src/isa/riscv64/mod.rs | 2 +- cranelift/codegen/src/isa/s390x/mod.rs | 2 +- cranelift/codegen/src/isa/x64/abi.rs | 80 +++++++++++++++++++ cranelift/codegen/src/isa/x64/mod.rs | 2 +- cranelift/codegen/src/machinst/abi.rs | 7 ++ cranelift/codegen/src/machinst/vcode.rs | 9 ++- 8 files changed, 98 insertions(+), 8 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index b7adf46f8676..b6134464954c 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -74,7 +74,7 @@ impl TargetIsa for AArch64Backend { ) -> CodegenResult { let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; - let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); + let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane)?; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rs index 66d0051e9e62..69ac4fd5f337 100644 --- a/cranelift/codegen/src/isa/pulley_shared/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs @@ -179,7 +179,7 @@ where let want_disasm = want_disasm || (cfg!(feature = "trace-log") && log::log_enabled!(log::Level::Debug)); - let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); + let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane)?; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs index 21b899d3b5c4..e8f8ca477405 100644 --- a/cranelift/codegen/src/isa/riscv64/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/mod.rs @@ -76,7 +76,7 @@ impl TargetIsa for Riscv64Backend { let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug); - let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); + let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane)?; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs index 1f2c2461ee02..9d92adb7a9f3 100644 --- a/cranelift/codegen/src/isa/s390x/mod.rs +++ b/cranelift/codegen/src/isa/s390x/mod.rs @@ -73,7 +73,7 @@ impl TargetIsa for S390xBackend { let flags = self.flags(); let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; - let emit_result = vcode.emit(®alloc_result, want_disasm, flags, ctrl_plane); + let emit_result = vcode.emit(®alloc_result, want_disasm, flags, ctrl_plane)?; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 7961044fb117..1cc398250a85 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -809,6 +809,30 @@ impl ABIMachineSpec for X64ABIMachineSpec { insts } + fn validate_frame_layout(frame_layout: &FrameLayout, guard_size: u32) -> CodegenResult<()> { + let rsp_relative: u64 = + frame_layout.active_size() as u64 + frame_layout.setup_area_size as u64; + let rbp_relative: u64 = + frame_layout.tail_args_size as u64 + frame_layout.setup_area_size as u64; + + // The stack-probe loop rounds the frame size up to the next guard-page + // boundary (align_to in emit.rs) before encoding it as a 32-bit immediate. + // A frame size just under the limit can be pushed over it by this rounding, + // so reserve a full guard-page margin. + let probe_margin = guard_size as u64; + + if rsp_relative.saturating_add(probe_margin) > i32::MAX as u64 + || rbp_relative > i32::MAX as u64 + { + return Err(crate::CodegenError::Unsupported( + "stack frame size exceeds the 2GB limit supported by the x64 backend \ + (see rust-lang/rustc_codegen_cranelift#1656)" + .to_owned(), + )); + } + Ok(()) + } + fn gen_memcpy Writable>( call_conv: isa::CallConv, dst: Reg, @@ -1343,3 +1367,59 @@ const fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { env } + +#[cfg(test)] +mod tests { + use super::*; + + fn make_frame_layout( + tail_args_size: u32, + setup_area_size: u32, + clobber_size: u32, + fixed_frame_storage_size: u32, + outgoing_args_size: u32, + ) -> FrameLayout { + FrameLayout { + word_bytes: 8, + incoming_args_size: tail_args_size, + tail_args_size, + setup_area_size, + clobber_size, + fixed_frame_storage_size, + stackslots_size: 0, + outgoing_args_size, + clobbered_callee_saves: Vec::new(), + function_calls: FunctionCalls::None, + } + } + + const GUARD_SIZE: u32 = 4096; + + #[test] + fn validate_frame_layout_accepts_normal_frame() { + let layout = make_frame_layout(64, 16, 64, 1024, 32); + assert!(X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE).is_ok()); + } + + #[test] + fn validate_frame_layout_rejects_oversized_fixed_storage() { + let layout = make_frame_layout(64, 16, 64, 2_147_483_647, 32); + let result = X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE); + assert!(result.is_err()); + } + + #[test] + fn validate_frame_layout_rejects_frame_pushed_over_by_guard_margin() { + let just_under = (i32::MAX as u32) - 2000; + let layout = make_frame_layout(64, 16, 64, just_under, 0); + let result = X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE); + assert!(result.is_err()); + } + + #[test] + fn validate_frame_layout_rejects_oversized_tail_args() { + let layout = make_frame_layout(2_147_483_647, 16, 0, 0, 0); + let result = X64ABIMachineSpec::validate_frame_layout(&layout, GUARD_SIZE); + assert!(result.is_err()); + } +} diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 15eb736f479d..96f6f3b5250c 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -82,7 +82,7 @@ impl TargetIsa for X64Backend { ) -> CodegenResult { let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; - let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); + let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane)?; let value_labels_ranges = emit_result.value_labels_ranges; let buffer = emit_result.buffer; diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index e467d8ccadac..d5682876d1f7 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -555,6 +555,13 @@ pub trait ABIMachineSpec { frame_layout: &FrameLayout, ) -> SmallVec<[Self::I; 16]>; + /// Validate that the computed frame layout is supported by this backend. + /// Default: always succeeds. Backends with encoding limits (e.g. 32-bit + /// immediates) should override this to reject layouts they cannot emit. + fn validate_frame_layout(_frame_layout: &FrameLayout, _guard_size: u32) -> CodegenResult<()> { + Ok(()) + } + /// Generate a memcpy invocation. Used to set up struct /// args. Takes `src`, `dst` as read-only inputs and passes a temporary /// allocator. diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index 24e82202a0d9..a5f463762683 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -739,7 +739,7 @@ impl VCode { want_disasm: bool, flags: &settings::Flags, ctrl_plane: &mut ControlPlane, - ) -> EmitResult + ) -> CodegenResult where I: VCodeInst, { @@ -785,6 +785,9 @@ impl VCode { function_calls, ); + let guard_size = 1u32 << flags.probestack_size_log2(); + I::ABIMachineSpec::validate_frame_layout(self.abi.frame_layout(), guard_size)?; + // Emit blocks. let mut cur_srcloc = None; let mut last_offset = None; @@ -1164,13 +1167,13 @@ impl VCode { // Store metadata about frame layout in the MachBuffer. buffer.set_frame_layout(self.abi.frame_slot_metadata()); - EmitResult { + Ok(EmitResult { buffer: buffer.finish(&self.constants, ctrl_plane), bb_offsets, bb_edges, disasm: if want_disasm { Some(disasm) } else { None }, value_labels_ranges, - } + }) } fn monotonize_inst_offsets(&self, inst_offsets: &mut [CodeOffset], func_body_len: u32) { From b94de39b8a419ce103cc7130a726737c7e460df0 Mon Sep 17 00:00:00 2001 From: Soumik15630m Date: Wed, 1 Jul 2026 17:54:09 +0530 Subject: [PATCH 2/2] Adding a generic check for allocation in compute_frame_layout --- cranelift/codegen/src/isa/x64/abi.rs | 6 +----- cranelift/codegen/src/machinst/abi.rs | 24 ++++++++++++++++++++---- cranelift/codegen/src/machinst/vcode.rs | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 1cc398250a85..397c82c91c9b 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -824,11 +824,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { if rsp_relative.saturating_add(probe_margin) > i32::MAX as u64 || rbp_relative > i32::MAX as u64 { - return Err(crate::CodegenError::Unsupported( - "stack frame size exceeds the 2GB limit supported by the x64 backend \ - (see rust-lang/rustc_codegen_cranelift#1656)" - .to_owned(), - )); + return Err(crate::CodegenError::ImplLimitExceeded); } Ok(()) } diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index d5682876d1f7..a40e0ae7bbcd 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -2211,12 +2211,12 @@ impl Callee { spillslots: usize, clobbered: Vec>, function_calls: FunctionCalls, - ) { + ) -> CodegenResult<()> { let bytes = M::word_bytes(); let total_stacksize = self.stackslots_size + bytes * spillslots as u32; let mask = M::stack_align(self.call_conv) - 1; - let total_stacksize = (total_stacksize + mask) & !mask; // 16-align the stack. - self.frame_layout = Some(M::compute_frame_layout( + let total_stacksize = (total_stacksize + mask) & !mask; + let frame_layout = M::compute_frame_layout( self.call_conv, &self.flags, self.signature(), @@ -2227,7 +2227,23 @@ impl Callee { self.stackslots_size, total_stacksize, self.outgoing_args_size, - )); + ); + + // FrameLayout's fields are all u32, so any backend can in principle + // overflow a 32-bit sum here; this generic check applies to all targets. + // x64 needs it's own specific check. + let total: u64 = frame_layout.incoming_args_size as u64 + + frame_layout.tail_args_size as u64 + + frame_layout.setup_area_size as u64 + + frame_layout.clobber_size as u64 + + frame_layout.fixed_frame_storage_size as u64 + + frame_layout.outgoing_args_size as u64; + if total > u32::MAX as u64 { + return Err(CodegenError::ImplLimitExceeded); + } + + self.frame_layout = Some(frame_layout); + Ok(()) } /// Generate a prologue, post-regalloc. diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index a5f463762683..07d67ad27024 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -783,7 +783,7 @@ impl VCode { regalloc.num_spillslots, clobbers, function_calls, - ); + )?; let guard_size = 1u32 << flags.probestack_size_log2(); I::ABIMachineSpec::validate_frame_layout(self.abi.frame_layout(), guard_size)?;