diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 816ebe3fcf3d9..eb189372112ea 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -6,6 +6,7 @@ use rustc_abi::{ RegKind, Size, X86Call, }; use rustc_codegen_ssa::MemFlags; +use rustc_codegen_ssa::common::RealPredicate; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; @@ -175,6 +176,9 @@ impl LlvmType for Reg { impl LlvmType for CastTarget { fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { + if self.x87_floating_point_stack { + return cx.type_x86_fp80(); + } let rest_ll_unit = self.rest.unit.llvm_type(cx); let rest_count = if self.rest.total == Size::ZERO { 0 @@ -324,6 +328,7 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ) { arg_abi.store_fn_arg(self, idx, dst) } + fn store_arg( &mut self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, @@ -332,6 +337,165 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ) { arg_abi.store(self, val, dst) } + + fn x87_lossless_float_to_fp_stack(&mut self, value: &'ll Value, no_undef: bool) -> &'ll Value { + // If value is (partially) uninitialized (e.g. when returning `MaybeUninit`) then e.g. + // branching on it could lead to undefined behaviour. To ensure that doesn't happen and that + // any initialized bytes within a partially uninitialized value survive the round trip, + // freeze the value. + let value = if no_undef { value } else { self.freeze(value) }; + // While we only need to manually convert sNaNs, all NaNs can be converted the same way and + // checking whether `value` is NaN only takes a single floating-point x86 instruction, + // whereas checking if it is a signalling NaNs requires bit operations. LLVM also generally + // won't know a non-constant `value` is not a sNaN but could be a qNaN, so being more + // general here doesn't prevent the branch from being optimised out in likely scenarios. + let is_nan = self.fcmp(RealPredicate::RealUNO, value, value); + let is_nan_block = self.append_sibling_block("float_pre_ret.is_nan"); + let is_not_nan_block = self.append_sibling_block("float_pre_ret.is_not_nan"); + let after_block = self.append_sibling_block("float_pre_ret.after"); + let dbg_loc = self.get_dbg_loc(); + self.cond_br(is_nan, is_nan_block, is_not_nan_block); + + self.switch_to_block(is_nan_block); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } + // This manually converts a NaN to x86_fp80 to avoid setting the quiet NaN bit of + // signalling NaNs. + let num_bits = self.float_width(self.val_ty(value)) as u64; + assert!( + num_bits == 32 || num_bits == 64, + "attempt to return float on x87 floating point stack with width {num_bits}" + ); + let bits_ty = self.type_ix(num_bits); + let bits = self.bitcast(value, bits_ty); + // The high 16 bits of an x86_fp80 are the exponent and sign (the sign is the highest + // bit) NaNs always have all bits of the exponent set to 1, so the only bit that is + // needed from `value` is the sign bit. + // Shift out the lower bits. + let exp_and_sign = self.lshr(bits, self.const_uint(bits_ty, num_bits - 16)); + // Set the exponent to all 1s. + let exp_and_sign = self.or(exp_and_sign, self.const_uint(bits_ty, 0x7FFF)); + // Shift the exponent and sign into position. + let exp_and_sign = self.zext(exp_and_sign, self.type_ix(80)); + let exp_and_sign = self.shl(exp_and_sign, self.const_uint_big(self.type_ix(80), 64)); + + // The fraction of the input NaN needs to be shifted left to just before x86_fp80's + // explicit integer bit. There's no need to manually set the integer bit itself, as it + // will be already set to 1 due to the all 1s exponent in the input NaN. + let (fraction_bits, fraction) = match num_bits { + 32 => (f32::MANTISSA_DIGITS - 1, self.zext(bits, self.type_i64())), + 64 => (f64::MANTISSA_DIGITS - 1, bits), + _ => bug!(), + }; + // Shift the fraction into position. + let fraction = self.shl(fraction, self.const_u64(63 - u64::from(fraction_bits))); + let fraction = self.zext(fraction, self.type_ix(80)); + + let is_nan_res = self.or(exp_and_sign, fraction); + let is_nan_res = self.bitcast(is_nan_res, self.type_x86_fp80()); + self.br(after_block); + + self.switch_to_block(is_not_nan_block); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } + let is_not_nan_res = self.fpext(value, self.type_x86_fp80()); + self.br(after_block); + + self.switch_to_block(after_block); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } + self.phi( + self.type_x86_fp80(), + &[is_nan_res, is_not_nan_res], + &[is_nan_block, is_not_nan_block], + ) + } + + fn x87_lossless_fp_stack_to_float( + &mut self, + value: &'ll Value, + float_type: &'ll Type, + no_undef: bool, + ) -> &'ll Value { + // If value is (partially) uninitialized (e.g. when returning `MaybeUninit`) then e.g. + // branching on it could lead to undefined behaviour. To ensure that doesn't happen and that + // any initialized bytes within a partially uninitialized value survive the round trip, + // freeze the value. + let value = if no_undef { value } else { self.freeze(value) }; + let num_bits = self.float_width(float_type) as u64; + let fraction_bits = u64::from(match num_bits { + 32 => f32::MANTISSA_DIGITS - 1, + 64 => f64::MANTISSA_DIGITS - 1, + _ => bug!("attempt to return float on x87 floating point stack with width {num_bits}"), + }); + let dest_bits_type = self.type_ix(num_bits); + // While we only need to manually convert sNaNs, all NaNs can be converted the same way and + // checking whether `value` is NaN only takes a single floating-point x86 instruction, + // whereas checking if it is a signalling NaNs requires bit operations. LLVM also generally + // won't know a non-constant `value` is not a sNaN but could be a qNaN, so being more + // general here doesn't prevent the branch from being optimised out in likely scenarios. + let is_nan = self.fcmp(RealPredicate::RealUNO, value, value); + let is_nan_block = self.append_sibling_block("float_post_ret.is_nan"); + let is_not_nan_block = self.append_sibling_block("float_post_ret.is_not_nan"); + let after_block = self.append_sibling_block("float_post_ret.after"); + let dbg_loc = self.get_dbg_loc(); + self.cond_br(is_nan, is_nan_block, is_not_nan_block); + + self.switch_to_block(is_nan_block); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } + // This block converts a NaN to `x86_fp80` manually to avoid setting the quiet NaN bit of + // signalling NaNs. + // We don't handle the "invalid operand" bitpatterns here (which are treated like NaNs) as + // they can't be generated by any post-80387 hardware, and the return value should have been + // converted from an actual `f32`/`f64`. Even on non-SSE targets current compilers don't + // seem to miscompile code so badly as to allow user-supplied `x86_fp80` "invalid operands" + // to be returned as `f32`/`f64`. Similarly, sNaNs are never produced by the hardware so we + // don't handle the case where the only fraction bits set are truncated, as that can never + // happen with sNaNs converted from `f32`s/`f64`s. + let bits = self.bitcast(value, self.type_ix(80)); + // Mask out the extra 1s in the exponent as `f32`/`f64` have less bits in their + // exponents than `x86_fp80`. + let exp_and_sign_bits = num_bits - fraction_bits; + let exp_and_sign_mask = u16::MAX << (16 - exp_and_sign_bits); + let exp_and_sign_mask = u128::from(exp_and_sign_mask) << 64; + let exp_and_sign = self.and(bits, self.const_uint_big(self.type_ix(80), exp_and_sign_mask)); + // Shift the exponent and sign into position + let exp_and_sign = self + .lshr(exp_and_sign, self.const_uint_big(self.type_ix(80), 80 - u128::from(num_bits))); + let exp_and_sign = self.trunc(exp_and_sign, dest_bits_type); + + // Truncate off the exponent and sign + let fraction = self.trunc(bits, self.type_i64()); + // Shift the fraction in to position. There's no need to mask out `x86_fp80`'s + // explicit integer bit as the fraction is right next to the exponent which is all + // 1s anyway. + let fraction = self.lshr(fraction, self.const_u64(63 - fraction_bits)); + let fraction = if num_bits != 64 { self.trunc(fraction, dest_bits_type) } else { fraction }; + + // Combine the parts into the resulting float + let is_nan_res = self.or(exp_and_sign, fraction); + let is_nan_res = self.bitcast(is_nan_res, float_type); + self.br(after_block); + + self.switch_to_block(is_not_nan_block); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } + // Use a regular floating point conversion when the value it not a NaN. + let is_not_nan_res = self.fptrunc(value, float_type); + self.br(after_block); + + self.switch_to_block(after_block); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } + self.phi(float_type, &[is_nan_res, is_not_nan_res], &[is_nan_block, is_not_nan_block]) + } } pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 87c941cdeb23a..33c7e0b4b1686 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1555,6 +1555,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } +impl<'ll> Builder<'_, 'll, '_> { + pub(crate) fn freeze(&mut self, value: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMBuildFreeze(self.llbuilder, value, UNNAMED) } + } +} + impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { fn get_static(&mut self, def_id: DefId) -> &'ll Value { // Forward to the `get_static` method of `CodegenCx` diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 05d3bd0b08b95..df17f67150eb9 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -940,6 +940,7 @@ unsafe extern "C" { // Operations on non-IEEE real types pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMX86FP80TypeInContext(C: &Context) -> &Type; // Operations on function types pub(crate) fn LLVMFunctionType<'a>( @@ -1596,6 +1597,11 @@ unsafe extern "C" { Index: c_uint, Name: *const c_char, ) -> &'a Value; + pub(crate) fn LLVMBuildFreeze<'a>( + B: &Builder<'a>, + Val: &'a Value, + Name: *const c_char, + ) -> &'a Value; // Atomic Operations pub(crate) fn LLVMBuildAtomicCmpXchg<'a>( diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 22d43f22e24a4..57aa3693edbb2 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -183,6 +183,10 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { pub(crate) fn type_bf16(&self) -> &'ll Type { unsafe { llvm::LLVMBFloatTypeInContext(self.llcx()) } } + + pub(crate) fn type_x86_fp80(&self) -> &'ll Type { + unsafe { llvm::LLVMX86FP80TypeInContext(self.llcx()) } + } } impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index afd9a88784c2f..9cd5035ecfafb 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -19,7 +19,7 @@ use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::{Span, Spanned}; -use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAbi, ArgAttribute, ArgAttributes, CastTarget, FnAbi, PassMode}; use tracing::{debug, info}; use super::operand::OperandRef; @@ -281,7 +281,9 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { // If the return value was retagged as it was stored, // then we might be in a different basic block now. // Update the cached block for `target` to point to this new - // block, where codegen will continue. + // block, where codegen will continue. Additionally, store_return() may have + // required a branch into a new codegen backend basic block (currently this occurs + // when `cast_target.x87_floating_point_stack` is set). fx.cached_llbbs[target] = CachedLlbb::Some(bx.llbb()); } MergingSucc::False @@ -2511,7 +2513,7 @@ fn load_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( align: Align, ) -> Bx::Value { let cast_ty = bx.cast_backend_type(cast); - if let Some(offset_from_start) = cast.rest_offset { + let value = if let Some(offset_from_start) = cast.rest_offset { assert_eq!(cast.prefix.len(), 1); assert_eq!(cast.rest.unit.size, cast.rest.total); let first_ty = bx.reg_backend_type(&cast.prefix[0]); @@ -2523,7 +2525,21 @@ fn load_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let res = bx.insert_value(res, first, 0); bx.insert_value(res, second, 1) } else { - bx.load(cast_ty, ptr, align) + let load_ty = if cast.x87_floating_point_stack { + match cast.rest.unit.size.bytes() { + 4 => bx.type_f32(), + 8 => bx.type_f64(), + _ => bug!(), + } + } else { + cast_ty + }; + bx.load(load_ty, ptr, align) + }; + if cast.x87_floating_point_stack { + bx.x87_lossless_float_to_fp_stack(value, cast.attrs.contains(ArgAttribute::NoUndef)) + } else { + value } } @@ -2534,6 +2550,20 @@ pub fn store_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( ptr: Bx::Value, align: Align, ) { + let value = if cast.x87_floating_point_stack { + let float_type = match cast.rest.unit.size.bytes() { + 4 => bx.type_f32(), + 8 => bx.type_f64(), + _ => bug!(), + }; + bx.x87_lossless_fp_stack_to_float( + value, + float_type, + cast.attrs.contains(ArgAttribute::NoUndef), + ) + } else { + value + }; if let Some(offset_from_start) = cast.rest_offset { assert_eq!(cast.prefix.len(), 1); assert_eq!(cast.rest.unit.size, cast.rest.total); diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 707eb3a6ee85d..5c1c1315289e4 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -157,6 +157,42 @@ pub trait ArgAbiBuilderMethods<'tcx>: BackendTypes { val: Self::Value, dst: PlaceRef<'tcx, Self::Value>, ); + /// Losslessly convert a `f32` or `f64` to a float to be returned on the x87 floating point + /// stack. Because `MaybeUninit` `f32`s/`f64`s are also returned on the floating point stack, + /// `value` being uninitialized must not cause undefined behaviour unless `true` is passed in + /// the `no_undef` argument. This method is used to avoid an LLVM bug where signalling NaNs get + /// quietened when being returned on the x87 stack on 32-bit x86. For more details, see: + /// * + /// * + fn x87_lossless_float_to_fp_stack( + &mut self, + value: Self::Value, + no_undef: bool, + ) -> Self::Value { + let _ = no_undef; + // Default to leaving the value unchanged. The backend can override this method if it needs + // extra codegen to avoid quietening signalling NaNs. + value + } + /// Losslessly convert a `f32` or `f64` from a float that was returned on the x87 floating point + /// stack. Because `MaybeUninit` `f32`s/`f64`s are also returned on the floating point stack, + /// `value` being uninitialized must not cause undefined behaviour unless `true` is passed in + /// the `no_undef` argument. This method is used to avoid an LLVM bug where signalling NaNs get + /// quietened when being returned on the x87 stack on 32-bit x86. For more details, see: + /// * + /// * + fn x87_lossless_fp_stack_to_float( + &mut self, + value: Self::Value, + float_type: Self::Type, + no_undef: bool, + ) -> Self::Value { + let _ = float_type; + let _ = no_undef; + // Default to leaving the value unchanged. The backend can override this method if it needs + // extra codegen to avoid quietening signalling NaNs. + value + } } pub trait TypeCodegenMethods<'tcx> = DerivedTypeCodegenMethods<'tcx> diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 26fedbd8a5481..887cb609cb09c 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -280,6 +280,12 @@ pub struct CastTarget { pub rest_offset: Option, pub rest: Uniform, pub attrs: ArgAttributes, + /// If `true`, indicates that this type will be passed on the x87 floating point stack on + /// 32-bit x86, changing the LLVM type to `x86_fp80`. This is needed as LLVM needs to generate + /// extra code to ensure that signalling NaNs are passed losslessly on the x87 floating point + /// stack. Only valid on return types on 32-bit x86 with a cast target of either `Reg::f32()` or + /// `Reg::f64()`. + pub x87_floating_point_stack: bool, } impl From for CastTarget { @@ -296,7 +302,13 @@ impl From for CastTarget { impl CastTarget { pub fn prefixed(prefix: ArrayVec, rest: Uniform) -> Self { - Self { prefix, rest_offset: None, rest, attrs: ArgAttributes::new() } + Self { + prefix, + rest_offset: None, + rest, + attrs: ArgAttributes::new(), + x87_floating_point_stack: false, + } } pub fn offset_pair(a: Reg, offset_from_start: Size, b: Reg) -> Self { @@ -307,6 +319,7 @@ impl CastTarget { rest_offset: Some(offset_from_start), rest: b.into(), attrs: ArgAttributes::new(), + x87_floating_point_stack: false, } } @@ -358,17 +371,20 @@ impl CastTarget { rest_offset: rest_offset_l, rest: rest_l, attrs: attrs_l, + x87_floating_point_stack: x87_l, } = self; let CastTarget { prefix: prefix_r, rest_offset: rest_offset_r, rest: rest_r, attrs: attrs_r, + x87_floating_point_stack: x87_r, } = other; prefix_l == prefix_r && rest_offset_l == rest_offset_r && rest_l == rest_r && attrs_l.eq_abi(attrs_r) + && x87_l == x87_r } } @@ -659,9 +675,8 @@ impl<'a, Ty> FnAbi<'a, Ty> { match &spec.arch { Arch::X86 => { let (flavor, regparm) = match abi { - ExternAbi::Fastcall { .. } | ExternAbi::Vectorcall { .. } => { - (x86::Flavor::FastcallOrVectorcall, None) - } + ExternAbi::Fastcall { .. } => (x86::Flavor::Fastcall, None), + ExternAbi::Vectorcall { .. } => (x86::Flavor::Vectorcall, None), ExternAbi::C { .. } | ExternAbi::Cdecl { .. } | ExternAbi::Stdcall { .. } => { (x86::Flavor::General, cx.x86_abi_opt().regparm) } diff --git a/compiler/rustc_target/src/callconv/x86.rs b/compiler/rustc_target/src/callconv/x86.rs index fd608fcf62919..f6ba80e141351 100644 --- a/compiler/rustc_target/src/callconv/x86.rs +++ b/compiler/rustc_target/src/callconv/x86.rs @@ -2,13 +2,38 @@ use rustc_abi::{ AddressSpace, Align, BackendRepr, Float, HasDataLayout, Primitive, Reg, RegKind, TyAndLayout, }; -use crate::callconv::{ArgAttribute, FnAbi, PassMode, TyAbiInterface}; +use crate::callconv::{ + ArgAbi, ArgAttribute, ArgAttributes, CastTarget, FnAbi, PassMode, TyAbiInterface, +}; use crate::spec::{HasTargetSpec, RustcAbi}; -#[derive(PartialEq)] +#[derive(Copy, Clone, PartialEq)] pub(crate) enum Flavor { General, - FastcallOrVectorcall, + Fastcall, + Vectorcall, +} + +pub(crate) fn pass_on_x87_floating_point_stack<'a, C, Ty>(cx: &C, arg_abi: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + let mut cast: CastTarget = match arg_abi.layout.size.bytes() { + 4 => Reg::f32().into(), + 8 => Reg::f64().into(), + _ => unreachable!("arg must be the size of a `f32` or `f64`"), + }; + cast.x87_floating_point_stack = true; + // Forward whether the argument is `NoUndef` or not to improve codegen. + cast.attrs = if let PassMode::Direct(attrs) = arg_abi.mode { + attrs + } else if super::layout_is_noundef(arg_abi.layout, cx) { + ArgAttribute::NoUndef.into() + } else { + ArgAttributes::new() + }; + arg_abi.mode = PassMode::Cast { pad_i32_count: 0, cast: Box::new(cast) }; } pub(crate) struct X86Options { @@ -23,6 +48,9 @@ where C: HasDataLayout + HasTargetSpec, { if !fn_abi.ret.is_ignore() { + // "vectorcall" returns floats in `xmm0`, and soft float also does not use the x87 stack. + let uses_x87_return = cx.target_spec().rustc_abi != Some(RustcAbi::Softfloat) + && opts.flavor != Flavor::Vectorcall; if fn_abi.ret.layout.is_aggregate() && fn_abi.ret.layout.is_sized() { // Returning a structure. Most often, this will use // a hidden first argument. On some platforms, though, @@ -44,6 +72,12 @@ where // float aggregates directly in a floating-point register. if fn_abi.ret.layout.is_single_fp_element(cx) { match fn_abi.ret.layout.size.bytes() { + // The calling convention passes float returns via the x87 stack. Tell the + // backend to convert to an `x86_fp80` manually to avoid LLVM quieting + // signalling NaNs when loading/storing to/from the x87 stack. + 4 | 8 if uses_x87_return => { + pass_on_x87_floating_point_stack(cx, &mut fn_abi.ret) + } 4 => fn_abi.ret.cast_to(Reg::f32()), 8 => fn_abi.ret.cast_to(Reg::f64()), _ => fn_abi.ret.make_indirect(), @@ -60,6 +94,11 @@ where } else { fn_abi.ret.make_indirect(); } + } else if uses_x87_return + && let BackendRepr::Scalar(scalar) = fn_abi.ret.layout.backend_repr + && matches!(scalar.primitive(), Primitive::Float(Float::F32 | Float::F64)) + { + pass_on_x87_floating_point_stack(cx, &mut fn_abi.ret); } else { fn_abi.ret.extend_integer_width_to(32); } @@ -143,7 +182,7 @@ pub(crate) fn fill_inregs<'a, Ty, C>( ) where Ty: TyAbiInterface<'a, C> + Copy, { - if opts.flavor != Flavor::FastcallOrVectorcall && opts.regparm.is_none_or(|x| x == 0) { + if opts.flavor == Flavor::General && opts.regparm.is_none_or(|x| x == 0) { return; } // Mark arguments as InReg like clang does it, diff --git a/compiler/rustc_target/src/callconv/x86_win32.rs b/compiler/rustc_target/src/callconv/x86_win32.rs index 9303711b7f6ad..22ebe49bd3078 100644 --- a/compiler/rustc_target/src/callconv/x86_win32.rs +++ b/compiler/rustc_target/src/callconv/x86_win32.rs @@ -1,7 +1,10 @@ -use rustc_abi::{Align, Float, HasDataLayout, Primitive, Reg, RegKind, TyAbiInterface}; +use rustc_abi::{ + Align, BackendRepr, Float, HasDataLayout, Primitive, Reg, RegKind, TyAbiInterface, +}; use crate::callconv::FnAbi; -use crate::spec::HasTargetSpec; +use crate::callconv::x86::Flavor; +use crate::spec::{HasTargetSpec, RustcAbi}; pub(crate) fn compute_abi_info<'a, Ty, C>( cx: &C, @@ -40,6 +43,12 @@ pub(crate) fn compute_abi_info<'a, Ty, C>( } else { fn_abi.ret.make_indirect(); } + } else if cx.target_spec().rustc_abi != Some(RustcAbi::Softfloat) + && opts.flavor != Flavor::Vectorcall + && let BackendRepr::Scalar(scalar) = fn_abi.ret.layout.backend_repr + && matches!(scalar.primitive(), Primitive::Float(Float::F32 | Float::F64)) + { + super::x86::pass_on_x87_floating_point_stack(cx, &mut fn_abi.ret); } else { fn_abi.ret.extend_integer_width_to(32); } diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index 7518ee9fabbbc..d36e684c36b5c 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -35,18 +35,15 @@ target | notes [`aarch64-apple-darwin`](platform-support/apple-darwin.md) | ARM64 macOS (11.0+, Big Sur+) [`aarch64-pc-windows-msvc`](platform-support/windows-msvc.md) | ARM64 Windows MSVC [`aarch64-unknown-linux-gnu`](platform-support/aarch64-unknown-linux-gnu.md) | ARM64 Linux (kernel 4.1+, glibc 2.17+) -[`i686-pc-windows-msvc`](platform-support/windows-msvc.md) | 32-bit MSVC (Windows 10+, Windows Server 2016+, Pentium 4) [^x86_32-floats-return-ABI] [^win32-msvc-alignment] -`i686-unknown-linux-gnu` | 32-bit Linux (kernel 3.2+, glibc 2.17+, Pentium 4) [^x86_32-floats-return-ABI] +[`i686-pc-windows-msvc`](platform-support/windows-msvc.md) | 32-bit MSVC (Windows 10+, Windows Server 2016+, Pentium 4) [^win32-msvc-alignment] +`i686-unknown-linux-gnu` | 32-bit Linux (kernel 3.2+, glibc 2.17+, Pentium 4) [`x86_64-pc-windows-gnu`](platform-support/windows-gnu.md) | 64-bit MinGW (Windows 10+, Windows Server 2016+) [`x86_64-pc-windows-msvc`](platform-support/windows-msvc.md) | 64-bit MSVC (Windows 10+, Windows Server 2016+) `x86_64-unknown-linux-gnu` | 64-bit Linux (kernel 3.2+, glibc 2.17+) -[^x86_32-floats-return-ABI]: Due to limitations of the C ABI, floating-point support on `i686` targets is non-compliant: floating-point return values are passed via an x87 register, so NaN payload bits can be lost. Functions with the default Rust ABI are not affected. See [issue #115567][x86-32-float-return-issue]. - [^win32-msvc-alignment]: Due to non-standard behavior of MSVC, native C code on this target can cause types with an alignment of more than 4 bytes to be incorrectly aligned to only 4 bytes (this affects, e.g., `u64` and `i64`). Rust applies some mitigations to reduce the impact of this issue, but this can still cause unsoundness due to unsafe code that (correctly) assumes that references are always properly aligned. See [issue #112480](https://github.com/rust-lang/rust/issues/112480). [77071]: https://github.com/rust-lang/rust/issues/77071 -[x86-32-float-return-issue]: https://github.com/rust-lang/rust/issues/115567 ## Tier 1 @@ -182,11 +179,11 @@ target | std | notes [`thumbv8r-none-eabihf`](platform-support/armv8r-none-eabihf.md) | * | Thumb-mode Bare Armv8-R, hardfloat `i586-unknown-linux-gnu` | ✓ | 32-bit Linux (kernel 3.2+, glibc 2.17, original Pentium) [^x86_32-floats-x87] `i586-unknown-linux-musl` | ✓ | 32-bit Linux (musl 1.2.5, original Pentium) [^x86_32-floats-x87] -[`i686-linux-android`](platform-support/android.md) | ✓ | 32-bit x86 Android ([Pentium 4 plus various extensions](https://developer.android.com/ndk/guides/abis.html#x86)) [^x86_32-floats-return-ABI] -[`i686-pc-windows-gnu`](platform-support/windows-gnu.md) | ✓ | 32-bit MinGW (Windows 10+, Windows Server 2016+, Pentium 4) [^x86_32-floats-return-ABI] [^win32-msvc-alignment] -[`i686-pc-windows-gnullvm`](platform-support/windows-gnullvm.md) | ✓ | 32-bit x86 MinGW (Windows 10+, Pentium 4), LLVM ABI [^x86_32-floats-return-ABI] -[`i686-unknown-freebsd`](platform-support/freebsd.md) | ✓ | 32-bit x86 FreeBSD (Pentium 4) [^x86_32-floats-return-ABI] -`i686-unknown-linux-musl` | ✓ | 32-bit Linux with musl 1.2.5 (Pentium 4) [^x86_32-floats-return-ABI] +[`i686-linux-android`](platform-support/android.md) | ✓ | 32-bit x86 Android ([Pentium 4 plus various extensions](https://developer.android.com/ndk/guides/abis.html#x86)) +[`i686-pc-windows-gnu`](platform-support/windows-gnu.md) | ✓ | 32-bit MinGW (Windows 10+, Windows Server 2016+, Pentium 4) [^win32-msvc-alignment] +[`i686-pc-windows-gnullvm`](platform-support/windows-gnullvm.md) | ✓ | 32-bit x86 MinGW (Windows 10+, Pentium 4), LLVM ABI +[`i686-unknown-freebsd`](platform-support/freebsd.md) | ✓ | 32-bit x86 FreeBSD (Pentium 4) +`i686-unknown-linux-musl` | ✓ | 32-bit Linux with musl 1.2.5 (Pentium 4) [`i686-unknown-uefi`](platform-support/unknown-uefi.md) | ? | 32-bit UEFI (Pentium 4, softfloat) [^win32-msvc-alignment] [`loongarch32-unknown-none`](platform-support/loongarch-none.md) | * | LoongArch32 Bare-metal (ILP32D ABI) [`loongarch32-unknown-none-softfloat`](platform-support/loongarch-none.md) | * | LoongArch32 Bare-metal (ILP32S ABI) @@ -337,22 +334,22 @@ target | std | host | notes [`hexagon-unknown-linux-musl`](platform-support/hexagon-unknown-linux-musl.md) | ✓ | | Hexagon Linux with musl 1.2.5 [`hexagon-unknown-none-elf`](platform-support/hexagon-unknown-none-elf.md)| * | | Bare Hexagon (v60+, HVX) [`hexagon-unknown-qurt`](platform-support/hexagon-unknown-qurt.md)| * | | Hexagon QuRT -[`i386-apple-ios`](platform-support/apple-ios.md) | ✓ | | 32-bit x86 iOS (Penryn) [^x86_32-floats-return-ABI] +[`i386-apple-ios`](platform-support/apple-ios.md) | ✓ | | 32-bit x86 iOS (Penryn) [`i586-unknown-netbsd`](platform-support/netbsd.md) | ✓ | | 32-bit x86 (original Pentium) [^x86_32-floats-x87] [`i586-unknown-redox`](platform-support/redox.md) | ✓ | | 32-bit x86 Redox OS (PentiumPro) [^x86_32-floats-x87] -[`i686-apple-darwin`](platform-support/apple-darwin.md) | ✓ | ✓ | 32-bit macOS (10.12+, Sierra+, Penryn) [^x86_32-floats-return-ABI] +[`i686-apple-darwin`](platform-support/apple-darwin.md) | ✓ | ✓ | 32-bit macOS (10.12+, Sierra+, Penryn) [`i686-oe-linux-gnu`](platform-support/oe-linux-gnu.md) | ✓ | | 32-bit x86 OpenEmbedded/Yocto Linux (GNU) -[`i686-pc-nto-qnx700`](platform-support/nto-qnx.md) | * | | 32-bit x86 QNX SDP 7.0 (Pentium 4) [^x86_32-floats-return-ABI] -`i686-unknown-haiku` | ✓ | ✓ | 32-bit Haiku (Pentium 4) [^x86_32-floats-return-ABI] +[`i686-pc-nto-qnx700`](platform-support/nto-qnx.md) | * | | 32-bit x86 QNX SDP 7.0 (Pentium 4) +`i686-unknown-haiku` | ✓ | ✓ | 32-bit Haiku (Pentium 4) [`i686-unknown-helenos`](platform-support/helenos.md) | ✓ | | HelenOS IA-32 (see docs for pending issues) -[`i686-unknown-hurd-gnu`](platform-support/hurd.md) | ✓ | ✓ | 32-bit GNU/Hurd (Pentium 4) [^x86_32-floats-return-ABI] -[`i686-unknown-netbsd`](platform-support/netbsd.md) | ✓ | ✓ | NetBSD/i386 (Pentium 4) [^x86_32-floats-return-ABI] -[`i686-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | 32-bit OpenBSD (Pentium 4) [^x86_32-floats-return-ABI] -`i686-uwp-windows-gnu` | ✓ | | [^x86_32-floats-return-ABI] -[`i686-uwp-windows-msvc`](platform-support/uwp-windows-msvc.md) | ✓ | | [^x86_32-floats-return-ABI] [^win32-msvc-alignment] -[`i686-win7-windows-gnu`](platform-support/win7-windows-gnu.md) | ✓ | | 32-bit Windows 7 support [^x86_32-floats-return-ABI] -[`i686-win7-windows-msvc`](platform-support/win7-windows-msvc.md) | ✓ | | 32-bit Windows 7 support [^x86_32-floats-return-ABI] [^win32-msvc-alignment] -[`i686-wrs-vxworks`](platform-support/vxworks.md) | ✓ | | [^x86_32-floats-return-ABI] +[`i686-unknown-hurd-gnu`](platform-support/hurd.md) | ✓ | ✓ | 32-bit GNU/Hurd (Pentium 4) +[`i686-unknown-netbsd`](platform-support/netbsd.md) | ✓ | ✓ | NetBSD/i386 (Pentium 4) +[`i686-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | 32-bit OpenBSD (Pentium 4) +`i686-uwp-windows-gnu` | ✓ | | +[`i686-uwp-windows-msvc`](platform-support/uwp-windows-msvc.md) | ✓ | | [^win32-msvc-alignment] +[`i686-win7-windows-gnu`](platform-support/win7-windows-gnu.md) | ✓ | | 32-bit Windows 7 support +[`i686-win7-windows-msvc`](platform-support/win7-windows-msvc.md) | ✓ | | 32-bit Windows 7 support [^win32-msvc-alignment] +[`i686-wrs-vxworks`](platform-support/vxworks.md) | ✓ | | [`loongarch64-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | | LoongArch64 OpenHarmony [`m68k-unknown-linux-gnu`](platform-support/m68k-unknown-linux-gnu.md) | ? | | Motorola 680x0 Linux [`m68k-unknown-none-elf`](platform-support/m68k-unknown-none-elf.md) | | | Motorola 680x0 diff --git a/tests/assembly-llvm/c-variadic/x86-linux.rs b/tests/assembly-llvm/c-variadic/x86-linux.rs index 70aa729aba7e8..c745e5af6b673 100644 --- a/tests/assembly-llvm/c-variadic/x86-linux.rs +++ b/tests/assembly-llvm/c-variadic/x86-linux.rs @@ -46,8 +46,10 @@ pub struct VaList<'a> { #[rustc_nounwind] pub const unsafe fn va_arg(ap: &mut VaList<'_>) -> T; +// Using the `"C"` ABI for the test function here would make the ASM significantly more verbose on +// 32-bit x86; and this test is testing `va_arg`, not the C return ABI. #[unsafe(no_mangle)] -unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 { +unsafe fn read_f64(ap: &mut VaList<'_>) -> f64 { // CHECK-LABEL: read_f64 // X86_64: mov ecx, dword ptr [rdi + 4] @@ -86,7 +88,7 @@ unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 { // I686-NEXT: mov ecx, dword ptr [eax] // I686-NEXT: lea edx, [ecx + 8] // I686-NEXT: mov dword ptr [eax], edx - // I686-NEXT: fld qword ptr [ecx] + // I686-NEXT: movsd xmm0, qword ptr [ecx] // I686-NEXT: ret va_arg(ap) } diff --git a/tests/assembly-llvm/x86-return-float-c.rs b/tests/assembly-llvm/x86-return-float-c.rs new file mode 100644 index 0000000000000..023518cc5bb15 --- /dev/null +++ b/tests/assembly-llvm/x86-return-float-c.rs @@ -0,0 +1,128 @@ +//@ assembly-output: emit-asm +//@ add-minicore +//@ revisions: linux windows-gnu windows-msvc +//@[linux] compile-flags: --target i686-unknown-linux-gnu +//@[linux] needs-llvm-components: x86 +//@[windows-gnu] compile-flags: --target i686-pc-windows-gnu +//@[windows-gnu] needs-llvm-components: x86 +//@[windows-msvc] compile-flags: --target i686-pc-windows-msvc +//@[windows-msvc] needs-llvm-components: x86 +//@ compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel + +#![feature(no_core)] +#![no_core] +#![crate_type = "lib"] + +extern crate minicore; +use minicore::*; + +// Tests that returning `f32` and `f64` with the "C" ABI on 32-bit x86 preserves signalling NaNs. + +// CHECK-LABEL: return_f32: +#[unsafe(no_mangle)] +pub extern "C" fn return_f32(x: f32) -> f32 { + // CHECK: movss [[XMM:.*]], dword ptr [{{esp|ebp}} + [[#]]] + // CHECK: ucomiss [[XMM]], [[XMM]] + // CHECK: jp [[NAN_LABEL:.*]] + // CHECK: movss dword ptr [esp + [[#OFFSET:]]], [[XMM]] + // CHECK: fld dword ptr [esp + [[#OFFSET]]] + // CHECK: ret + // CHECK: [[NAN_LABEL]]: + // CHECK: movd [[BITS:.*]], [[XMM]] + // CHECK: mov dword ptr [esp + [[#OFFSET:]]], 0 + // CHECK: mov e[[SIGN_AND_EXP:.*]], [[BITS]] + // CHECK: shl [[BITS]], 8 + // CHECK: shr e[[SIGN_AND_EXP]], 16 + // CHECK: mov dword ptr [esp + [[#OFFSET+4]]], [[BITS]] + // CHECK: or e[[SIGN_AND_EXP]], 32767 + // CHECK: mov word ptr [esp + [[#OFFSET+8]]], [[SIGN_AND_EXP]] + // CHECK: fld tbyte ptr [esp + [[#OFFSET]]] + // CHECK: ret + x +} + +// CHECK-LABEL: return_f64: +#[unsafe(no_mangle)] +pub extern "C" fn return_f64(x: f64) -> f64 { + // CHECK: movsd [[XMM:.*]], qword ptr [{{esp|ebp}} + {{.*}}] + // CHECK: ucomisd [[XMM]], [[XMM]] + // CHECK: jp [[NAN_LABEL:.*]] + // CHECK: movsd qword ptr [esp + [[#OFFSET:]]], [[XMM]] + // CHECK: fld qword ptr [esp + [[#OFFSET]]] + // CHECK: ret + // CHECK: [[NAN_LABEL]]: + // CHECK: movsd qword ptr [esp + [[#OFFSET:]]], [[XMM]] + // CHECK: mov [[HIGH:.*]], dword ptr [esp + [[#OFFSET+4]]] + // CHECK: mov [[LOW:.*]], dword ptr [esp + [[#OFFSET]]] + // CHECK: mov e[[SIGN_AND_EXP:.*]], [[HIGH]] + // CHECK: shld [[HIGH]], [[LOW]], 11 + // CHECK: shl [[LOW]], 11 + // CHECK: shr e[[SIGN_AND_EXP]], 16 + // CHECK: mov dword ptr [esp + 4], [[HIGH]] + // CHECK: mov dword ptr [esp], [[LOW]] + // CHECK: or e[[SIGN_AND_EXP]], 32767 + // CHECK: mov word ptr [esp + 8], [[SIGN_AND_EXP]] + // CHECK: fld tbyte ptr [esp] + // CHECK: ret + x +} + +// CHECK-LABEL: call_f32: +#[unsafe(no_mangle)] +pub unsafe fn call_f32(x: &mut f32) { + extern "C" { + fn get_f32() -> f32; + } + // CHECK: mov [[PTR:.*]], dword ptr [{{esp|ebp}} + {{.*}}] + // CHECK: call {{()|_}}get_f32 + // CHECK: fucomi st, st(0) + // CHECK: jp [[NAN_LABEL:.*]] + // CHECK: fstp dword ptr [esp + [[#OFFSET:]]] + // CHECK: movd [[XMM:.*]], dword ptr [esp + [[#OFFSET]]] + // CHECK: [[RET_LABEL:.*]]: + // CHECK: movd dword ptr [[[PTR]]], [[XMM]] + // CHECK: ret + // CHECK: [[NAN_LABEL]]: + // CHECK: fstp tbyte ptr [esp + [[#OFFSET:]]] + // CHECK: mov [[SIGN_AND_EXP:.*]], dword ptr [esp + [[#OFFSET+8]]] + // CHECK: mov [[BITS:.*]], dword ptr [esp + [[#OFFSET+4]]] + // CHECK: and [[SIGN_AND_EXP]], -128 + // CHECK: shr [[BITS]], 8 + // CHECK: shl [[SIGN_AND_EXP]], 16 + // CHECK: or [[BITS]], [[SIGN_AND_EXP]] + // CHECK: movd [[XMM]], [[BITS]] + // CHECK: jmp [[RET_LABEL]] + *x = get_f32(); +} + +// CHECK-LABEL: call_f64: +#[unsafe(no_mangle)] +pub unsafe fn call_f64(x: &mut f64) { + extern "C" { + fn get_f64() -> f64; + } + // CHECK: mov [[PTR:.*]], dword ptr [{{esp|ebp}} + {{.*}}] + // CHECK: call {{()|_}}get_f64 + // CHECK: fucomi st, st(0) + // CHECK: jp [[NAN_LABEL:.*]] + // CHECK: fstp qword ptr [esp + [[#OFFSET:]]] + // CHECK: movq [[XMM:.*]], qword ptr [esp + [[#OFFSET]]] + // CHECK: [[RET_LABEL:.*]]: + // CHECK: movq qword ptr [[[PTR]]], [[XMM]] + // CHECK: ret + // CHECK: [[NAN_LABEL]]: + // CHECK: fstp tbyte ptr [esp] + // CHECK: mov [[SIGN_AND_EXP:.*]], dword ptr [esp + 8] + // CHECK: mov [[LOW:.*]], dword ptr [esp] + // CHECK: mov [[HIGH:.*]], dword ptr [esp + 4] + // CHECK: and [[SIGN_AND_EXP]], -16 + // CHECK: shrd [[LOW]], [[HIGH]], 11 + // CHECK: shr [[HIGH]], 11 + // CHECK: shl [[SIGN_AND_EXP]], 16 + // CHECK: movd [[XMM]], [[LOW]] + // CHECK: or [[HIGH]], [[SIGN_AND_EXP]] + // CHECK: movd [[XMM_HIGH:.*]], [[HIGH]] + // CHECK: punpckldq [[XMM]], [[XMM_HIGH]] + // CHECK: jmp [[RET_LABEL]] + *x = get_f64(); +} diff --git a/tests/auxiliary/rust_test_helpers.c b/tests/auxiliary/rust_test_helpers.c index cd10d6b98ca7b..1794d5bbcf81f 100644 --- a/tests/auxiliary/rust_test_helpers.c +++ b/tests/auxiliary/rust_test_helpers.c @@ -17,11 +17,30 @@ rust_dbg_extern_identity_u64(uint64_t u) { return u; } +float +rust_dbg_extern_identity_float(float u) { + return u; +} + double rust_dbg_extern_identity_double(double u) { return u; } +typedef float (*float_callback)(float); + +void +rust_dbg_extern_call_float(float_callback f, float u, float* res) { + *res = f(u); +} + +typedef double (*double_callback)(double); + +void +rust_dbg_extern_call_double(double_callback f, double u, double* res) { + *res = f(u); +} + char rust_dbg_extern_identity_u8(char u) { return u; diff --git a/tests/codegen-llvm/float/x86-return-float-c.rs b/tests/codegen-llvm/float/x86-return-float-c.rs new file mode 100644 index 0000000000000..9ece1c45ddcef --- /dev/null +++ b/tests/codegen-llvm/float/x86-return-float-c.rs @@ -0,0 +1,117 @@ +//@ assembly-output: emit-asm +//@ add-minicore +//@ revisions: linux windows-gnu windows-msvc +//@[linux] compile-flags: --target i686-unknown-linux-gnu +//@[linux] needs-llvm-components: x86 +//@[windows-gnu] compile-flags: --target i686-pc-windows-gnu +//@[windows-gnu] needs-llvm-components: x86 +//@[windows-msvc] compile-flags: --target i686-pc-windows-msvc +//@[windows-msvc] needs-llvm-components: x86 +// We want to test LLVM optimisations, so disabled MIR optimisations. +//@ compile-flags: -Copt-level=3 -Zmir-opt-level=0 -Zmerge-functions=disabled + +#![feature(no_core)] +#![no_core] +#![crate_type = "lib"] + +extern crate minicore; +use minicore::*; + +// Check undef values don't cause undefined behaviour. + +// CHECK-LABEL: @return_undef_f32() +// CHECK-NOT: unreachable +// CHECK: ret x86_fp80 +#[unsafe(no_mangle)] +extern "C" fn return_undef_f32() -> MaybeUninit { + MaybeUninit::uninit() +} + +// CHECK-LABEL: @return_undef_f64() +// CHECK-NOT: unreachable +// CHECK: ret x86_fp80 +#[unsafe(no_mangle)] +extern "C" fn return_undef_f64() -> MaybeUninit { + MaybeUninit::uninit() +} + +// CHECK-LABEL: @call_undef_f32() +// CHECK-NOT: unreachable +// CHECK: ret x86_fp80 +#[unsafe(no_mangle)] +extern "C" fn call_undef_f32() -> MaybeUninit { + return_undef_f32() +} + +// CHECK-LABEL: @call_undef_f64() +// CHECK-NOT: unreachable +// CHECK: ret x86_fp80 +#[unsafe(no_mangle)] +extern "C" fn call_undef_f64() -> MaybeUninit { + return_undef_f64() +} + +// Check LLVM can still propogate constants + +// CHECK-LABEL: @return_constant_f32() +// FIXME: The hexadecimal can be removed once LLVM 22 is dropped. +// CHECK: ret x86_fp80 {{1\.500000e\+00|0xK3FFFC000000000000000}} +#[unsafe(no_mangle)] +extern "C" fn return_constant_f32() -> f32 { + 1.5 +} + +// CHECK-LABEL: @return_constant_f64() +// CHECK: ret x86_fp80 {{2\.500000e\+00|0xK4000A000000000000000}} +#[unsafe(no_mangle)] +extern "C" fn return_constant_f64() -> f64 { + 2.5 +} + +// CHECK-LABEL: @call_constant_f32() +// CHECK: ret x86_fp80 {{1\.500000e\+00|0xK3FFFC000000000000000}} +#[unsafe(no_mangle)] +extern "C" fn call_constant_f32() -> f32 { + return_constant_f32() +} + +// CHECK-LABEL: @call_constant_f64() +// CHECK: ret x86_fp80 {{2\.500000e\+00|0xK4000A000000000000000}} +#[unsafe(no_mangle)] +extern "C" fn call_constant_f64() -> f64 { + return_constant_f64() +} + +// Check that LLVM can optimise away the NaN branch when the value is guaranteed to be non-NaN. + +// CHECK-LABEL: @return_non_nan_f32(i16 {{.*}} %x) +// CHECK: %0 = sitofp i16 %x to x86_fp80 +// CHECK-NEXT: ret x86_fp80 %0 +#[unsafe(no_mangle)] +extern "C" fn return_non_nan_f32(x: i16) -> f32 { + x as _ +} + +// CHECK-LABEL: @return_non_nan_f64(i32 {{.*}} %x) +// CHECK: %0 = sitofp i32 %x to x86_fp80 +// CHECK-NEXT: ret x86_fp80 %0 +#[unsafe(no_mangle)] +extern "C" fn return_non_nan_f64(x: i32) -> f64 { + x as _ +} + +// CHECK-LABEL: @call_non_nan_f32(i16 {{.*}} %x) +// CHECK: %0 = sitofp i16 %x to x86_fp80 +// CHECK-NEXT: ret x86_fp80 %0 +#[unsafe(no_mangle)] +extern "C" fn call_non_nan_f32(x: i16) -> f32 { + return_non_nan_f32(x) +} + +// CHECK-LABEL: @call_non_nan_f64(i32 {{.*}} %x) +// CHECK: %0 = sitofp i32 %x to x86_fp80 +// CHECK-NEXT: ret x86_fp80 %0 +#[unsafe(no_mangle)] +extern "C" fn call_non_nan_f64(x: i32) -> f64 { + return_non_nan_f64(x) +} diff --git a/tests/codegen-llvm/reg-struct-return.rs b/tests/codegen-llvm/reg-struct-return.rs index 52a1e174dfe6b..a6f5313a5b99b 100644 --- a/tests/codegen-llvm/reg-struct-return.rs +++ b/tests/codegen-llvm/reg-struct-return.rs @@ -187,14 +187,14 @@ pub mod tests { FooFloat1 { x: 1.0, y: 1.0 } } - // ENABLED: double @f15() + // ENABLED: x86_fp80 @f15() // DISABLED: void @f15(ptr {{.*}}sret #[no_mangle] pub extern "C" fn f15() -> FooFloat2 { FooFloat2 { x: 1.0 } } - // ENABLED: float @f16() + // ENABLED: x86_fp80 @f16() // DISABLED: void @f16(ptr {{.*}}sret #[no_mangle] pub extern "C" fn f16() -> FooFloat3 { diff --git a/tests/codegen-llvm/repr/transparent.rs b/tests/codegen-llvm/repr/transparent.rs index 29b627462a4d9..ba3f4b22d7727 100644 --- a/tests/codegen-llvm/repr/transparent.rs +++ b/tests/codegen-llvm/repr/transparent.rs @@ -2,6 +2,7 @@ //@ ignore-riscv64 riscv64 has an i128 type used with test_Vector //@ ignore-s390x s390x with default march passes vector types per reference //@ ignore-loongarch64 see codegen/loongarch-abi for loongarch function call tests +//@ ignore-x86 32-bit x86 manually returns `f32`/`f64` as `x86_fp80` to avoid LLVM codegen bugs. // This codegen test embeds assumptions about how certain "C" psABIs are handled // so it doesn't apply to all architectures or even all OS diff --git a/tests/ui/abi/numbers-arithmetic/float-ffi.rs b/tests/ui/abi/numbers-arithmetic/float-ffi.rs new file mode 100644 index 0000000000000..2809a48570a00 --- /dev/null +++ b/tests/ui/abi/numbers-arithmetic/float-ffi.rs @@ -0,0 +1,115 @@ +//@ run-pass +//@ compile-flags: -Copt-level=0 + +// Test that floats roundtrip correctly through C functions. + +use std::mem::MaybeUninit; + +#[link(name = "rust_test_helpers", kind = "static")] +unsafe extern "C" { + safe fn rust_dbg_extern_identity_float(x: f32) -> f32; + safe fn rust_dbg_extern_identity_double(x: f64) -> f64; + safe fn rust_dbg_extern_call_float( + f: extern "C" fn(f32) -> f32, + x: f32, + res: &mut MaybeUninit, + ); + safe fn rust_dbg_extern_call_double( + f: extern "C" fn(f64) -> f64, + x: f64, + res: &mut MaybeUninit, + ); +} + +fn main() { + let bits_f32 = std::hint::black_box( + const { + [ + 4.2_f32.to_bits(), + f32::INFINITY.to_bits(), + f32::NEG_INFINITY.to_bits(), + f32::NAN.to_bits(), + // These two masks cover all the fraction bits. One of them is a signalling NaN, the + // other is quiet. + // Similar to the masks in `test_float_bits_conv` in library/std/src/f32/tests.rs + f32::NAN.to_bits() ^ 0x002A_AAAA, + f32::NAN.to_bits() ^ 0x0055_5555, + // Same as above but with the sign bit flipped. + f32::NAN.to_bits() ^ 0x802A_AAAA, + f32::NAN.to_bits() ^ 0x8055_5555, + ] + }, + ); + for bits in bits_f32 { + let res = rust_dbg_extern_identity_float(f32::from_bits(bits)).to_bits(); + // On 32-bit x86, `f32`s are returned on the x87 stack. Allow the result to have been + // quietened, as the C compiler might not have ensured that placing the float on the + // the stack is lossless. + let quiet_bit = 0x0040_0000; + assert!( + res == bits + || (cfg!(target_arch = "x86") + && f32::from_bits(bits).is_nan() + && bits | quiet_bit == res) + ); + let res = unsafe { + let mut res = MaybeUninit::uninit(); + rust_dbg_extern_call_float(identity, f32::from_bits(bits), &mut res); + res.assume_init().to_bits() + }; + assert!( + res == bits + || (cfg!(target_arch = "x86") + && f32::from_bits(bits).is_nan() + && bits | quiet_bit == res) + ); + } + + let bits_f64 = std::hint::black_box( + const { + [ + 4.2_f64.to_bits(), + f64::INFINITY.to_bits(), + f64::NEG_INFINITY.to_bits(), + f64::NAN.to_bits(), + // These two masks cover all the fraction bits. One of them is a signalling NaN, the + // other is quiet. + // Similar to the masks in `test_float_bits_conv` in library/std/src/f64/tests.rs + f64::NAN.to_bits() ^ 0x000A_AAAA_AAAA_AAAA, + f64::NAN.to_bits() ^ 0x0005_5555_5555_5555, + // Same as above but with the sign bit flipped. + f64::NAN.to_bits() ^ 0x800A_AAAA_AAAA_AAAA, + f64::NAN.to_bits() ^ 0x8005_5555_5555_5555, + ] + }, + ); + for bits in bits_f64 { + let res = rust_dbg_extern_identity_double(f64::from_bits(bits)).to_bits(); + // On 32-bit x86, `f32`s are returned on the x87 stack. Allow the result to have been + // quietened, as the C compiler might not have ensured that placing the float on the + // the stack is lossless. + let quiet_bit = 0x0008_0000_0000_0000; + assert!( + res == bits + || (cfg!(target_arch = "x86") + && f64::from_bits(bits).is_nan() + && bits | quiet_bit == res) + ); + let res = unsafe { + let mut res = MaybeUninit::uninit(); + rust_dbg_extern_call_double(identity, f64::from_bits(bits), &mut res); + res.assume_init().to_bits() + }; + assert!( + res == bits + || (cfg!(target_arch = "x86") + && f64::from_bits(bits).is_nan() + && bits | quiet_bit == res) + ); + } +} + +#[inline(never)] +extern "C" fn identity(x: T) -> T { + x +} diff --git a/tests/ui/abi/numbers-arithmetic/return-float.rs b/tests/ui/abi/numbers-arithmetic/return-float.rs index 66a6d66911d3c..d003609474b7e 100644 --- a/tests/ui/abi/numbers-arithmetic/return-float.rs +++ b/tests/ui/abi/numbers-arithmetic/return-float.rs @@ -1,61 +1,197 @@ //@ run-pass -//@ compile-flags: -Copt-level=0 +//@ revisions: no-opts cg-opts-only all-opts +// No optimisations: Functions won't be inlined, so the machine-level ABI will be tested. +//@[no-opts] compile-flags: -Copt-level=0 +// Codegen optimisations only: Functions will only be inlined by the codegen backend. +//@[cg-opts-only] compile-flags: -Copt-level=3 -Zmir-opt-level=0 +// All optimisations: Functions will be inlined in MIR or the codegen backend. +//@[all-opts] compile-flags: -Copt-level=3 + +#![cfg_attr(all(target_arch = "x86", target_feature = "sse2"), feature(abi_vectorcall))] // Test that floats (in particular signalling NaNs) are losslessly returned from functions. +use std::mem::MaybeUninit; + fn main() { - // FIXME(#114479): LLVM miscompiles loading and storing `f32` and `f64` when SSE is disabled on - // x86. - if cfg!(not(all(target_arch = "x86", not(target_feature = "sse2")))) { - let bits_f32 = std::hint::black_box([ - 4.2_f32.to_bits(), - f32::INFINITY.to_bits(), - f32::NEG_INFINITY.to_bits(), - f32::NAN.to_bits(), - // These two masks cover all the mantissa bits. One of them is a signalling NaN, the - // other is quiet. - // Similar to the masks in `test_float_bits_conv` in library/std/src/f32/tests.rs - f32::NAN.to_bits() ^ 0x002A_AAAA, - f32::NAN.to_bits() ^ 0x0055_5555, - // Same as above but with the sign bit flipped. - f32::NAN.to_bits() ^ 0x802A_AAAA, - f32::NAN.to_bits() ^ 0x8055_5555, - ]); - for bits in bits_f32 { - assert_eq!(identity(f32::from_bits(bits)).to_bits(), bits); - // Test types that are returned as scalar pairs. - assert_eq!(identity((f32::from_bits(bits), 42)).0.to_bits(), bits); - assert_eq!(identity((42, f32::from_bits(bits))).1.to_bits(), bits); - let (a, b) = identity((f32::from_bits(bits), f32::from_bits(bits))); - assert_eq!((a.to_bits(), b.to_bits()), (bits, bits)); - } - - let bits_f64 = std::hint::black_box([ - 4.2_f64.to_bits(), - f64::INFINITY.to_bits(), - f64::NEG_INFINITY.to_bits(), - f64::NAN.to_bits(), - // These two masks cover all the mantissa bits. One of them is a signalling NaN, the - // other is quiet. - // Similar to the masks in `test_float_bits_conv` in library/std/src/f64/tests.rs - f64::NAN.to_bits() ^ 0x000A_AAAA_AAAA_AAAA, - f64::NAN.to_bits() ^ 0x0005_5555_5555_5555, - // Same as above but with the sign bit flipped. - f64::NAN.to_bits() ^ 0x800A_AAAA_AAAA_AAAA, - f64::NAN.to_bits() ^ 0x8005_5555_5555_5555, - ]); - for bits in bits_f64 { - assert_eq!(identity(f64::from_bits(bits)).to_bits(), bits); - // Test types that are returned as scalar pairs. - assert_eq!(identity((f64::from_bits(bits), 42)).0.to_bits(), bits); - assert_eq!(identity((42, f64::from_bits(bits))).1.to_bits(), bits); - let (a, b) = identity((f64::from_bits(bits), f64::from_bits(bits))); - assert_eq!((a.to_bits(), b.to_bits()), (bits, bits)); - } + let bits_f32 = std::hint::black_box( + const { + [ + 4.2_f32.to_bits(), + f32::INFINITY.to_bits(), + f32::NEG_INFINITY.to_bits(), + f32::NAN.to_bits(), + // These two masks cover all the mantissa bits. One of them is a signalling NaN, the + // other is quiet. + // Similar to the masks in `test_float_bits_conv` in library/std/src/f32/tests.rs + f32::NAN.to_bits() ^ 0x002A_AAAA, + f32::NAN.to_bits() ^ 0x0055_5555, + // Same as above but with the sign bit flipped. + f32::NAN.to_bits() ^ 0x802A_AAAA, + f32::NAN.to_bits() ^ 0x8055_5555, + ] + }, + ); + let bits_f64 = std::hint::black_box( + const { + [ + 4.2_f64.to_bits(), + f64::INFINITY.to_bits(), + f64::NEG_INFINITY.to_bits(), + f64::NAN.to_bits(), + // These two masks cover all the mantissa bits. One of them is a signalling NaN, the + // other is quiet. + // Similar to the masks in `test_float_bits_conv` in library/std/src/f64/tests.rs + f64::NAN.to_bits() ^ 0x000A_AAAA_AAAA_AAAA, + f64::NAN.to_bits() ^ 0x0005_5555_5555_5555, + // Same as above but with the sign bit flipped. + f64::NAN.to_bits() ^ 0x800A_AAAA_AAAA_AAAA, + f64::NAN.to_bits() ^ 0x8005_5555_5555_5555, + ] + }, + ); + + #[repr(C)] + struct Struct(T); + + // FIXME(#114479): LLVM miscompiles loading and storing `f32` and `f64` when SSE(2) is disabled + // on x86 (i586-* targets), meaning signalling NaNs get quietened in the middle of function + // bodies. + let check_f32 = |res: u32, bits: u32, abi: &str, what: &str| { + let quiet_bit = 0x0040_0000; + assert!( + res == bits + || (cfg!(all(target_arch = "x86", not(target_feature = "sse"))) + && f32::from_bits(bits).is_nan() + && bits | quiet_bit == res), + "{res:x} != {bits:x} {} f32 {}", + abi, + what + ); + }; + let check_f64 = |res: u64, bits: u64, abi: &str, what: &str| { + let quiet_bit = 0x0008_0000_0000_0000; + assert!( + res == bits + || (cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) + && f64::from_bits(bits).is_nan() + && bits | quiet_bit == res), + "{res:x} != {bits:x} {} f64 {}", + abi, + what + ); + }; + + macro_rules! abi_check { + ($($abi:literal),+ $(,)?) => { + $({ + #[cfg_attr(no_opts, inline(never))] + extern $abi fn identity(x: T) -> T { + x + } + + for bits in bits_f32 { + check_f32(identity(f32::from_bits(bits)).to_bits(), bits, $abi, "direct"); + // Check single element structs are returned correctly too. + check_f32( + identity(Struct(f32::from_bits(bits))).0.to_bits(), + bits, + $abi, + "struct", + ); + // Ensure value is still preserved when wrapped in a MaybeUninit. + unsafe { + check_f32( + identity(MaybeUninit::new(f32::from_bits(bits))) + .assume_init() + .to_bits(), + bits, + $abi, + "MaybeUninit", + ); + check_f32( + identity(MaybeUninit::new(Struct(f32::from_bits(bits)))) + .assume_init() + .0 + .to_bits(), + bits, + $abi, + "struct MaybeUninit", + ); + } + } + for bits in bits_f64 { + check_f64(identity(f64::from_bits(bits)).to_bits(), bits, $abi, "direct"); + check_f64( + identity(Struct(f64::from_bits(bits))).0.to_bits(), + bits, + $abi, + "struct", + ); + unsafe { + check_f64( + identity(MaybeUninit::new(f64::from_bits(bits))) + .assume_init() + .to_bits(), + bits, + $abi, + "MaybeUninit", + ); + check_f64( + identity(MaybeUninit::new(Struct(f64::from_bits(bits)))) + .assume_init() + .0 + .to_bits(), + bits, + $abi, + "struct MaybeUninit", + ); + } + } + // Returning `MaybeUninit::uninit()` must not cause undefined behaviour. + std::hint::black_box(identity(MaybeUninit::::uninit())); + std::hint::black_box(identity(MaybeUninit::>::uninit())); + std::hint::black_box(identity(MaybeUninit::::uninit())); + std::hint::black_box(identity(MaybeUninit::>::uninit())); + })* + }; + } + abi_check!("Rust", "C", "C-unwind", "system", "system-unwind"); + // Test some extra platform-specific ABIs on 32-bit x86 as that is the platform where signaling + // NaNs often used to get quietened (and still do get quietened when SSE2 is disabled). + #[cfg(target_arch = "x86")] + abi_check!( + "cdecl", + "cdecl-unwind", + "efiapi", + "fastcall", + "fastcall-unwind", + "stdcall", + "stdcall-unwind", + "thiscall", + "thiscall-unwind", + ); + #[cfg(all(target_arch = "x86", target_feature = "sse2"))] + abi_check!("vectorcall", "vectorcall-unwind"); + + // Test types that are returned as scalar pairs. + #[cfg_attr(no_opts, inline(never))] + fn identity(x: T) -> T { + x } -} -#[inline(never)] -fn identity(x: T) -> T { - x + for bits in bits_f32 { + check_f32(identity((f32::from_bits(bits), 42)).0.to_bits(), bits, "Rust", "tuple.0"); + check_f32(identity((42, f32::from_bits(bits))).1.to_bits(), bits, "Rust", "tuple.1"); + let (a, b) = identity((f32::from_bits(bits), f32::from_bits(bits))); + check_f32(a.to_bits(), bits, "Rust", "pair.0"); + check_f32(b.to_bits(), bits, "Rust", "pair.1"); + } + for bits in bits_f64 { + check_f64(identity((f64::from_bits(bits), 42)).0.to_bits(), bits, "Rust", "tuple.0"); + check_f64(identity((42, f64::from_bits(bits))).1.to_bits(), bits, "Rust", "tuple.1"); + let (a, b) = identity((f64::from_bits(bits), f64::from_bits(bits))); + check_f64(a.to_bits(), bits, "Rust", "pair.0"); + check_f64(b.to_bits(), bits, "Rust", "pair.1"); + } } diff --git a/tests/ui/abi/pass-indirectly-attr.stderr b/tests/ui/abi/pass-indirectly-attr.stderr index efeec0d86982b..82217bfa6ad17 100644 --- a/tests/ui/abi/pass-indirectly-attr.stderr +++ b/tests/ui/abi/pass-indirectly-attr.stderr @@ -139,6 +139,7 @@ error: fn_abi_of(extern_rust) = FnAbi { pointee_size: Size(0 bytes), pointee_align: None, }, + x87_floating_point_stack: false, }, }, },