Adding a generic check for allocation in compute_frame_layout

Author: soumik15630m

cranelift/codegen/src/isa/x64/abi.rs

@@ -824,11 +824,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         if rsp_relative.saturating_add(probe_margin) > i32::MAX as u64
             || rbp_relative > i32::MAX as u64
         {
-            return Err(crate::CodegenError::Unsupported(
-                "stack frame size exceeds the 2GB limit supported by the x64 backend \
-             (see rust-lang/rustc_codegen_cranelift#1656)"
-                    .to_owned(),
-            ));
+            return Err(crate::CodegenError::ImplLimitExceeded);
         }
         Ok(())
     }

cranelift/codegen/src/machinst/abi.rs

@@ -2211,12 +2211,12 @@ impl<M: ABIMachineSpec> Callee<M> {
         spillslots: usize,
         clobbered: Vec<Writable<RealReg>>,
         function_calls: FunctionCalls,
-    ) {
+    ) -> CodegenResult<()> {
         let bytes = M::word_bytes();
         let total_stacksize = self.stackslots_size + bytes * spillslots as u32;
         let mask = M::stack_align(self.call_conv) - 1;
-        let total_stacksize = (total_stacksize + mask) & !mask; // 16-align the stack.
-        self.frame_layout = Some(M::compute_frame_layout(
+        let total_stacksize = (total_stacksize + mask) & !mask;
+        let frame_layout = M::compute_frame_layout(
             self.call_conv,
             &self.flags,
             self.signature(),
@@ -2227,7 +2227,23 @@ impl<M: ABIMachineSpec> Callee<M> {
             self.stackslots_size,
             total_stacksize,
             self.outgoing_args_size,
-        ));
+        );
+
+        // FrameLayout's fields are all u32, so any backend can in principle
+        // overflow a 32-bit sum here; this generic check applies to all targets.
+        // x64 needs it's own specific check.
+        let total: u64 = frame_layout.incoming_args_size as u64
+            + frame_layout.tail_args_size as u64
+            + frame_layout.setup_area_size as u64
+            + frame_layout.clobber_size as u64
+            + frame_layout.fixed_frame_storage_size as u64
+            + frame_layout.outgoing_args_size as u64;
+        if total > u32::MAX as u64 {
+            return Err(CodegenError::ImplLimitExceeded);
+        }
+
+        self.frame_layout = Some(frame_layout);
+        Ok(())
     }
 
     /// Generate a prologue, post-regalloc.

cranelift/codegen/src/machinst/vcode.rs

@@ -783,7 +783,7 @@ impl<I: VCodeInst> VCode<I> {
             regalloc.num_spillslots,
             clobbers,
             function_calls,
-        );
+        )?;
 
         let guard_size = 1u32 << flags.probestack_size_log2();
         I::ABIMachineSpec::validate_frame_layout(self.abi.frame_layout(), guard_size)?;