diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 88049d67964b1..83f2bcdd3959c 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -18,7 +18,7 @@ use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::{ BackendTypes, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - LayoutTypeCodegenMethods, OverflowOp, StaticBuilderMethods, + LayoutTypeCodegenMethods, OverflowOp, ReturnSlot, StaticBuilderMethods, }; use rustc_data_structures::fx::FxHashSet; use rustc_middle::bug; @@ -608,6 +608,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, @@ -618,7 +619,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let current_block = self.block; self.block = try_block; - let call = self.call(typ, fn_attrs, fn_abi, func, args, None, instance); // FIXME(antoyo): use funclet here? + // FIXME(antoyo): use funclet here? + let call = self.call(typ, fn_attrs, fn_abi, func, return_slot, args, None, instance); self.block = current_block; let return_value = @@ -646,13 +648,14 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>, instance: Option>, ) -> RValue<'gcc> { - let call_site = self.call(typ, fn_attrs, fn_abi, func, args, None, instance); + let call_site = self.call(typ, fn_attrs, fn_abi, func, return_slot, args, None, instance); let condition = self.context.new_rvalue_from_int(self.bool_type, 1); self.llbb().end_with_conditional(self.location, condition, then, catch); if let Some(_fn_abi) = fn_abi { @@ -1779,19 +1782,30 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], funclet: Option<&Funclet>, _instance: Option>, ) -> RValue<'gcc> { + // FIXME: change this in the `rustc_codegen_gcc` repo after the sync, to use the `libgccjit` indirect return suppport. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { + let mut args = args.to_vec(); + // Prepend the indirect return pointer + args.insert(0, sret_ptr); + args + } + }; // FIXME(antoyo): remove when having a proper API. let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { // FIXME(antoyo): remove when the API supports a different type for functions. let func: Function<'gcc> = self.cx.rvalue_as_function(func); - self.function_call(func, args, funclet) + self.function_call(func, &args, funclet) } else { // If it's a not function that was defined, it's a function pointer. - self.function_ptr_call(typ, fn_abi, func, args, funclet) + self.function_ptr_call(typ, fn_abi, func, &args, funclet) }; if let Some(_fn_abi) = fn_abi { // FIXME(bjorn3): Apply function attributes @@ -1805,6 +1819,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _fn_attrs: Option<&CodegenFnAttrs>, _fn_abi: &FnAbi<'tcx, Ty<'tcx>>, _llfn: Self::Value, + _return_slot: ReturnSlot, _args: &[Self::Value], _funclet: Option<&Self::Funclet>, _instance: Option>, diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 5550d22b33aa3..06713016cedbe 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -16,7 +16,7 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, + IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, ReturnSlot, }; use rustc_codegen_ssa::{MemFlags, RetagInfo}; use rustc_data_structures::fx::FxHashSet; @@ -655,7 +655,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } // FIXME directly use the llvm intrinsic adjustment functions here - let llret = self.call(fn_ty, None, None, fn_ptr, &call_args, None, None); + let llret = + self.call(fn_ty, None, None, fn_ptr, ReturnSlot::Direct, &call_args, None, None); if is_cleanup { self.apply_attrs_to_cleanup_callsite(llret); } @@ -666,7 +667,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; - self.call(self.type_void(), None, None, func, &[], None, None); + self.call(self.type_void(), None, None, func, ReturnSlot::Direct, &[], None, None); } fn assume(&mut self, value: Self::Value) { @@ -1347,7 +1348,7 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( let param_type = bx.u8_type.make_pointer(); let fn_type = bx.context.new_function_pointer_type(None, bx.type_void(), &[param_type], false); - bx.call(fn_type, None, None, try_func, &[data], None, None); + bx.call(fn_type, None, None, try_func, ReturnSlot::Direct, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. OperandValue::Immediate(bx.const_bool(false)).store(bx, dest); @@ -1420,21 +1421,41 @@ fn codegen_gnu_try<'gcc, 'tcx>( let zero = bx.cx.context.new_rvalue_zero(bx.int_type); let ptr = bx.cx.context.new_call(None, eh_pointer_builtin, &[zero]); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); // NOTE: the blocks must be filled before adding the try/catch, otherwise gcc will not // generate a try/catch. // FIXME(antoyo): add a check in the libgccjit API to prevent this. bx.switch_to_block(current_block); - bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + then, + catch, + None, + None, + ); }); let func = unsafe { std::mem::transmute::, RValue<'gcc>>(func) }; // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, func, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + func, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); OperandValue::Immediate(ret).store(bx, dest); } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index c22318c0ec8a7..8e5750a2998bc 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -586,9 +586,20 @@ pub(crate) fn inline_asm_call<'ll>( assert!(catch_funclet.is_none()); bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None) } else if let Some((catch, funclet)) = catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None) + bx.invoke( + fty, + None, + None, + v, + ReturnSlot::Direct, + inputs, + dest.unwrap(), + catch, + funclet, + None, + ) } else { - bx.call(fty, None, None, v, inputs, None, None) + bx.call(fty, None, None, v, ReturnSlot::Direct, inputs, None, None) }; // Store mark in a metadata node so we can map LLVM errors diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index dae8b2d17e0e1..9d4602e49968d 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -454,15 +454,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, + return_slot: ReturnSlot<&'ll Value>, args: &[&'ll Value], then: &'ll BasicBlock, catch: &'ll BasicBlock, funclet: Option<&Funclet<'ll>>, instance: Option>, ) -> &'ll Value { + // If this function returns indirectly (`PassMode::Indirect`), + // the `return_slot` should be the first argument. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { + let mut args = args.to_vec(); + args.insert(0, sret_ptr); + args + } + }; debug!("invoke {:?} with args ({:?})", llfn, args); - - let args = self.check_call("invoke", llty, llfn, args); + let args = self.check_call("invoke", llty, llfn, &args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1463,13 +1473,23 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, + return_slot: ReturnSlot<&'ll Value>, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, callee_instance: Option>, ) -> &'ll Value { + // If this function returns indirectly (`PassMode::Indirect`), + // the `return_slot` should be the first argument. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { + let mut args = args.to_vec(); + args.insert(0, sret_ptr); + args + } + }; debug!("call {:?} with args ({:?})", llfn, args); - - let args = self.check_call("call", llty, llfn, args); + let args = self.check_call("call", llty, llfn, &args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1530,12 +1550,21 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, ) { - let call = - self.call(llty, caller_attrs, Some(fn_abi), llfn, args, funclet, callee_instance); + let call = self.call( + llty, + caller_attrs, + Some(fn_abi), + llfn, + return_slot, + args, + funclet, + callee_instance, + ); llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail); match &fn_abi.ret.mode { @@ -1875,7 +1904,8 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { args: &[&'ll Value], ) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params); - self.call(ty, None, None, f, args, None, None) + // No LLVM intrinsic returns its data indirectly (via `sret`). + self.call(ty, None, None, f, ReturnSlot::Direct, args, None, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) { @@ -2091,6 +2121,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { None, None, ubsan_handler, + ReturnSlot::Direct, &[diag_data, function_address, self.const_usize(0)], None, None, diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 8cefd8dc489a9..3831f3b0912fd 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -6,7 +6,7 @@ use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::mir::IntrinsicResult; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceValue; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot}; use rustc_data_structures::thin_vec::ThinVec; use rustc_hir::attrs::RustcAutodiff; use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; @@ -372,7 +372,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( crate::typetree::add_tt(&bx, fn_to_diff, fnc_tree); } - let call = bx.call(enzyme_ty, None, None, ad_fn, &args, None, None); + let call = bx.call(enzyme_ty, None, None, ad_fn, ReturnSlot::Direct, &args, None, None); let fn_ret_ty = bx.cx.val_ty(call); if fn_ret_ty == bx.cx.type_void() || fn_ret_ty == bx.cx.type_struct(&[], false) { diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index e2ec20226e3ce..a84ca02cc3b18 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -6,7 +6,7 @@ use rustc_abi::Align; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot}; use rustc_middle::bug; use rustc_middle::ty::offload_meta::{MappingFlags, OffloadMetadata, OffloadSize}; @@ -681,7 +681,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( let num_args = cx.get_const_i32(num_args); let args = vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; - builder.call(fn_ty, None, None, fn_to_call, &args, None, None); + builder.call(fn_ty, None, None, fn_to_call, ReturnSlot::Direct, &args, None, None); } // Step 2) @@ -718,7 +718,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( let device_id = builder.sext(device_id, cx.type_i64()); let args = vec![s_ident_t, device_id, num_workgroups, threads_per_block, region_id, a5]; - builder.call(tgt_target_kernel_ty, None, None, tgt_decl, &args, None, None); + builder.call(tgt_target_kernel_ty, None, None, tgt_decl, ReturnSlot::Direct, &args, None, None); // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) // Step 4) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 1844a8e5c0bca..be384689be39d 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -245,7 +245,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::offload_get_num_devices => { let (fn_decl, fn_ty) = declare_omp_get_num_devices(self.cx); - let llval = self.call(fn_ty, None, None, fn_decl, &[], None, None); + let llval = + self.call(fn_ty, None, None, fn_decl, ReturnSlot::Direct, &[], None, None); return IntrinsicResult::Operand(OperandValue::Immediate(llval)); } @@ -1354,7 +1355,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( ) -> &'ll Value { if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.call(try_func_ty, None, None, try_func, &[data], None, None); + bx.call(try_func_ty, None, None, try_func, ReturnSlot::Direct, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. bx.const_bool(false) @@ -1452,7 +1453,18 @@ fn codegen_msvc_try<'ll, 'tcx>( let ptr_align = bx.tcx().data_layout.pointer_align().abi; let slot = bx.alloca(ptr_size, ptr_align); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + normal, + catchswitch, + None, + None, + ); bx.switch_to_block(normal); bx.ret(bx.const_bool(false)); @@ -1500,7 +1512,16 @@ fn codegen_msvc_try<'ll, 'tcx>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_ptr(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + ReturnSlot::Direct, + &[data, ptr], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -1508,7 +1529,16 @@ fn codegen_msvc_try<'ll, 'tcx>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_ptr()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + ReturnSlot::Direct, + &[data, null], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -1517,7 +1547,16 @@ fn codegen_msvc_try<'ll, 'tcx>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); ret } @@ -1564,7 +1603,18 @@ fn codegen_wasm_try<'ll, 'tcx>( // } // let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + normal, + catchswitch, + None, + None, + ); bx.switch_to_block(normal); bx.ret(bx.const_bool(false)); @@ -1580,7 +1630,16 @@ fn codegen_wasm_try<'ll, 'tcx>( let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + ReturnSlot::Direct, + &[data, ptr], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -1589,7 +1648,16 @@ fn codegen_wasm_try<'ll, 'tcx>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); ret } @@ -1630,7 +1698,18 @@ fn codegen_gnu_try<'ll, 'tcx>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + then, + catch, + None, + None, + ); bx.switch_to_block(then); bx.ret(bx.const_bool(false)); @@ -1648,13 +1727,22 @@ fn codegen_gnu_try<'ll, 'tcx>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); ret } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index d67156c6cfa39..e0b1df0bb632c 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -198,12 +198,21 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { args.push(llvm::get_param(alias_lldecl, index)); } + // For an indirect return, the alias's own first parameter is the + // caller-provided return slot: forward it to the aliasee as such. + let (return_slot, args) = if fn_abi.ret.is_indirect() { + let (sret_ptr, rest) = args.split_first().unwrap(); + (ReturnSlot::Indirect(*sret_ptr), rest) + } else { + (ReturnSlot::Direct, &args[..]) + }; let call = start_bx.call( fn_ty, Some(attrs), Some(fn_abi), aliasee, - &args, + return_slot, + args, None, Some(aliasee_instance), ); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 8dd129f45cc5a..d909316194566 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -593,7 +593,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( ) }; - let result = bx.call(start_ty, None, None, start_fn, &args, None, instance); + let result = + bx.call(start_ty, None, None, start_fn, ReturnSlot::Direct, &args, None, instance); if cx.sess().target.os == Os::Uefi { bx.ret(result); } else { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index afd9a88784c2f..c3d66b6df2b81 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -165,12 +165,16 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional /// return destination `destination` and the unwind action `unwind`. + /// The `return_slot` is [`ReturnSlot::Indirect`] for functions returning + /// via `PassMode::Indirect`, and points to a buffer where the return value + /// shall be stored. fn do_call>( &self, fx: &mut FunctionCx<'a, 'tcx, Bx>, bx: &mut Bx, fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, fn_ptr: Bx::Value, + return_slot: ReturnSlot, llargs: &[Bx::Value], destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>, mut unwind: mir::UnwindAction, @@ -244,8 +248,23 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } }; + debug_assert_eq!( + return_slot.is_indirect(), + fn_abi.ret.is_indirect(), + "a return slot must be provided if and only if the return is `PassMode::Indirect`", + ); + if kind == CallKind::Tail { - bx.tail_call(fn_ty, caller_attrs, fn_abi, fn_ptr, llargs, self.funclet(fx), instance); + bx.tail_call( + fn_ty, + caller_attrs, + fn_abi, + fn_ptr, + return_slot, + llargs, + self.funclet(fx), + instance, + ); return MergingSucc::False; } @@ -260,6 +279,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, + return_slot, llargs, ret_llbb, unwind_block, @@ -291,6 +311,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, + return_slot, llargs, self.funclet(fx), instance, @@ -718,6 +739,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, drop_fn, + ReturnSlot::Direct, args, Some((ReturnDest::Nothing, target)), unwind, @@ -822,6 +844,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + ReturnSlot::Direct, &args, None, unwind, @@ -853,6 +876,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + ReturnSlot::Direct, &[], None, mir::UnwindAction::Unreachable, @@ -922,6 +946,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + ReturnSlot::Direct, &[msg.0, msg.1], target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), unwind, @@ -1202,21 +1227,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // We still need to call `make_return_dest` even if there's no `target`, since // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited, // and `make_return_dest` adds the return-place indirect pointer to `llargs`. - let destination = match kind { + let (destination, return_slot) = match kind { CallKind::Normal => { - let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs); - target.map(|target| (return_dest, target)) + let (return_dest, return_slot) = + self.make_return_dest(bx, destination, &fn_abi.ret); + (target.map(|target| (return_dest, target)), return_slot) } CallKind::Tail => { - if fn_abi.ret.is_indirect() { - match self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs) { - ReturnDest::Nothing => {} + let return_slot = if fn_abi.ret.is_indirect() { + match self.make_return_dest(bx, destination, &fn_abi.ret) { + (ReturnDest::Nothing, return_slot) => return_slot, _ => bug!( "tail calls to functions with indirect returns cannot store into a destination" ), } - } - None + } else { + ReturnSlot::Direct + }; + (None, return_slot) } }; @@ -1441,6 +1469,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, fn_ptr, + return_slot, &llargs, destination, unwind, @@ -2360,7 +2389,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { let fn_ty = bx.fn_decl_backend_type(fn_abi); - let llret = bx.call(fn_ty, None, Some(fn_abi), fn_ptr, &[], funclet.as_ref(), None); + let llret = bx.call( + fn_ty, + None, + Some(fn_abi), + fn_ptr, + ReturnSlot::Direct, + &[], + funclet.as_ref(), + None, + ); bx.apply_attrs_to_cleanup_callsite(llret); } @@ -2396,11 +2434,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &mut Bx, dest: mir::Place<'tcx>, fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, - llargs: &mut Vec, - ) -> ReturnDest<'tcx, Bx::Value> { + ) -> (ReturnDest<'tcx, Bx::Value>, ReturnSlot) { // If the return is ignored, we can just return a do-nothing `ReturnDest`. if fn_ret.is_ignore() { - return ReturnDest::Nothing; + return (ReturnDest::Nothing, ReturnSlot::Direct); } let dest = if let Some(index) = dest.as_local() { match self.locals[index] { @@ -2414,10 +2451,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // but the calling convention has an indirect return. let tmp = PlaceRef::alloca(bx, fn_ret.layout); tmp.storage_live(bx); - llargs.push(tmp.val.llval); - ReturnDest::IndirectOperand(tmp, index) + ( + ReturnDest::IndirectOperand(tmp, index), + ReturnSlot::Indirect(tmp.val.llval), + ) } else { - ReturnDest::DirectOperand(index) + (ReturnDest::DirectOperand(index), ReturnSlot::Direct) }; } LocalRef::Operand(_) => { @@ -2437,10 +2476,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // to create a temporary. span_bug!(self.mir.span, "can't directly store to unaligned value"); } - llargs.push(dest.val.llval); - ReturnDest::Nothing + (ReturnDest::Nothing, ReturnSlot::Indirect(dest.val.llval)) } else { - ReturnDest::Store(dest) + (ReturnDest::Store(dest), ReturnSlot::Direct) } } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 344a4834862e4..6278020eabecd 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -779,6 +779,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_attrs.as_deref(), Some(fn_abi), fn_ptr, + ReturnSlot::Direct, &[], None, Some(instance), diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 0ef0179eac6c4..e286681c293ee 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -78,6 +78,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /* fn_attrs */ None, Some(fn_abi), llfn, + ReturnSlot::Direct, // we know the ABI here &[msg.0, msg.1], None, None, diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index cb0209a0ae369..b7b694922bcfa 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -34,6 +34,20 @@ pub enum OverflowOp { Mul, } +/// The location of the return value for the call. +#[derive(Copy, Clone, Debug)] +pub enum ReturnSlot { + Direct, + /// The return value will be passed via sret (e.g. `PassMode::Indirect`). + Indirect(V), +} + +impl ReturnSlot { + pub fn is_indirect(&self) -> bool { + matches!(self, ReturnSlot::Indirect(_)) + } +} + pub trait BuilderMethods<'a, 'tcx>: Sized + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> @@ -135,6 +149,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], then: Self::BasicBlock, catch: Self::BasicBlock, @@ -638,16 +653,20 @@ pub trait BuilderMethods<'a, 'tcx>: /// The typical case that they are None is during the codegen of intrinsics and lang-items, /// as those are "fake functions" with only a trivial ABI if any, et cetera. /// + /// `return_slot` must be `ReturnSlot::Indirect` if an argument uses `PassMode::Indirect`. + /// /// ## Return /// - /// Must return the value the function will return so it can be written to the destination, - /// assuming the function does not explicitly pass the destination as a pointer in `args`. + /// Must return the value the function will return so it can be written to the destination. + /// For calls with an indirect return, the returned value is meaningless and must not be + /// used: the return value lives in the return slot. fn call( &mut self, llty: Self::FunctionSignature, caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, fn_val: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, @@ -659,6 +678,7 @@ pub trait BuilderMethods<'a, 'tcx>: caller_attrs: Option<&CodegenFnAttrs>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index f46d07ea5008e..1f013cb122070 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -36,7 +36,7 @@ pub use self::asm::{ AsmBuilderMethods, AsmCodegenMethods, GlobalAsmOperandRef, InlineAsmOperandRef, }; pub use self::backend::{BackendTypes, CodegenBackend, ExtraBackendMethods}; -pub use self::builder::{BuilderMethods, OverflowOp}; +pub use self::builder::{BuilderMethods, OverflowOp, ReturnSlot}; pub use self::consts::ConstCodegenMethods; pub use self::coverageinfo::CoverageInfoBuilderMethods; pub use self::debuginfo::{DebugInfoBuilderMethods, DebugInfoCodegenMethods};