diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 09a3cf4d182f8..e0653fe4e3956 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -603,9 +603,13 @@ pub(crate) fn check_intrinsic_type( sym::cold_path => (0, 0, vec![], tcx.types.unit), sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)), + sym::read_field_via_copy => (2, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(1)), sym::write_via_move => { (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit) } + sym::write_field_via_move => { + (2, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(1)], tcx.types.unit) + } sym::write_box_via_move => { let t = param(0); let maybe_uninit_t = Ty::new_maybe_uninit(tcx, t); diff --git a/compiler/rustc_mir_build/src/builder/expr/into.rs b/compiler/rustc_mir_build/src/builder/expr/into.rs index ecc14fe887f24..4757cf71198b1 100644 --- a/compiler/rustc_mir_build/src/builder/expr/into.rs +++ b/compiler/rustc_mir_build/src/builder/expr/into.rs @@ -401,14 +401,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ExprKind::Call { ty, fun, ref args, .. } if let ty::FnDef(def_id, generic_args) = *ty.kind() && let Some(intrinsic) = this.tcx.intrinsic(def_id) - && matches!(intrinsic.name, sym::write_via_move | sym::write_box_via_move) => + && matches!( + intrinsic.name, + sym::write_via_move | sym::write_field_via_move | sym::write_box_via_move + ) => { // We still have to evaluate the callee expression as normal (but we don't care // about its result). let _fun = unpack!(block = this.as_local_operand(block, fun)); match intrinsic.name { - sym::write_via_move => { + sym::write_via_move | sym::write_field_via_move => { // `write_via_move(ptr, val)` becomes `*ptr = val` but without any dropping. // The destination must have unit type (so we don't actually have to store anything @@ -424,7 +427,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { span_bug!(expr_span, "invalid write_via_move call") }; let ptr_deref = ptr.project_deeper(&[ProjectionElem::Deref], this.tcx); - this.expr_into_dest(ptr_deref, block, val) + let dst_place = if intrinsic.name == sym::write_field_via_move { + let field_ty = generic_args[1].expect_ty(); + let field_idx = generic_args[2].expect_const(); + let field_idx = field_idx.try_to_leaf().unwrap().to_u32(); + ptr_deref.project_deeper( + &[PlaceElem::Field(FieldIdx::from_u32(field_idx), field_ty)], + this.tcx, + ) + } else { + ptr_deref + }; + this.expr_into_dest(dst_place, block, val) } sym::write_box_via_move => { // The signature is: diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index fe53d301c5574..48731b688645f 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -1,5 +1,6 @@ //! Lowers intrinsic calls +use rustc_abi::FieldIdx; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, span_bug}; @@ -148,7 +149,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { )); terminator.kind = TerminatorKind::Goto { target }; } - sym::read_via_copy => { + sym::read_via_copy | sym::read_field_via_copy => { let Ok([arg]) = take_array(args) else { span_bug!(terminator.source_info.span, "Wrong number of arguments"); }; @@ -162,6 +163,17 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { "Only passing a local is supported" ); }; + let derefed_place = if intrinsic.name == sym::read_field_via_copy { + let field_ty = generic_args[1].expect_ty(); + let field_idx = generic_args[2].expect_const(); + let field_idx = field_idx.try_to_leaf().unwrap().to_u32(); + derefed_place.project_deeper( + &[PlaceElem::Field(FieldIdx::from_u32(field_idx), field_ty)], + tcx, + ) + } else { + derefed_place + }; // Add new statement at the end of the block that does the read, and patch // up the terminator. block.statements.push(Statement::new( diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index c2c7041c663df..0af7dea37c5a5 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1649,6 +1649,7 @@ symbols! { raw_identifiers, raw_ref_op, re_rebalance_coherence, + read_field_via_copy, read_via_copy, readonly, realloc, @@ -2348,6 +2349,7 @@ symbols! { wreg, write_box_via_move, write_bytes, + write_field_via_move, write_fmt, write_macro, write_str, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index a714bfa098c1d..97a0fc57c8af7 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -2202,7 +2202,7 @@ pub const fn carryless_mul(a: T, b: T) -> T { a.carryless_mul(b) } -/// This is an implementation detail of [`crate::ptr::read`] and should +/// This is an implementation detail of [`ptr::read`] and should /// not be used anywhere else. See its comments for why this exists. /// /// This intrinsic can *only* be called where the pointer is a local without @@ -2213,7 +2213,7 @@ pub const fn carryless_mul(a: T, b: T) -> T { #[rustc_intrinsic] pub const unsafe fn read_via_copy(ptr: *const T) -> T; -/// This is an implementation detail of [`crate::ptr::write`] and should +/// This is an implementation detail of [`ptr::write`] and should /// not be used anywhere else. See its comments for why this exists. /// /// This intrinsic can *only* be called where the pointer is a local without @@ -2224,6 +2224,24 @@ pub const unsafe fn read_via_copy(ptr: *const T) -> T; #[rustc_intrinsic] pub const unsafe fn write_via_move(ptr: *mut T, value: T); +/// Lowers to `RET = copy (*ptr)._FIELD_IDX` in MIR. +/// +/// This is currently used only for [`ptr::read_unaligned`] so that it can emit +/// a `packed` load, and shouldn't be used elsewhere. +#[rustc_intrinsic_const_stable_indirect] +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn read_field_via_copy(ptr: *const T) -> F; + +/// Lowers to `(*ptr)._FIELD_IDX = move value` in MIR. +/// +/// This is currently used only for [`ptr::write_unaligned`] so that it can emit +/// a `packed` store, and shouldn't be used elsewhere. +#[rustc_intrinsic_const_stable_indirect] +#[rustc_nounwind] +#[rustc_intrinsic] +pub const unsafe fn write_field_via_move(ptr: *mut T, value: F); + /// Returns the value of the discriminant for the variant in 'v'; /// if `T` has no discriminant, returns `0`. /// diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 593011edecf27..f7d3bb42743a7 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -1810,17 +1810,12 @@ pub const unsafe fn read(src: *const T) -> T { #[track_caller] #[rustc_diagnostic_item = "ptr_read_unaligned"] pub const unsafe fn read_unaligned(src: *const T) -> T { - let mut tmp = MaybeUninit::::uninit(); - // SAFETY: the caller must guarantee that `src` is valid for reads. - // `src` cannot overlap `tmp` because `tmp` was just allocated on - // the stack as a separate allocation. - // - // Also, since we just wrote a valid value into `tmp`, it is guaranteed - // to be properly initialized. - unsafe { - copy_nonoverlapping(src as *const u8, tmp.as_mut_ptr() as *mut u8, size_of::()); - tmp.assume_init() - } + let src = src.cast::>(); + // Always true because it's `repr(C, packed)` + const { assert!(mem::offset_of!(Packed, 0) == 0) }; + // SAFETY: reading through a packed place reads the same memory because + // the field offset is zero, just with a lower alignment. + unsafe { crate::intrinsics::read_field_via_copy::, T, 0>(src) } } /// Overwrites a memory location with the given value without reading or @@ -2012,14 +2007,12 @@ pub const unsafe fn write(dst: *mut T, src: T) { #[rustc_diagnostic_item = "ptr_write_unaligned"] #[track_caller] pub const unsafe fn write_unaligned(dst: *mut T, src: T) { - // SAFETY: the caller must guarantee that `dst` is valid for writes. - // `dst` cannot overlap `src` because the caller has mutable access - // to `dst` while `src` is owned by this function. - unsafe { - copy_nonoverlapping((&raw const src) as *const u8, dst as *mut u8, size_of::()); - // We are calling the intrinsic directly to avoid function calls in the generated code. - intrinsics::forget(src); - } + let dst = dst.cast::>(); + // Always true because it's `repr(C, packed)` + const { assert!(mem::offset_of!(Packed, 0) == 0) }; + // SAFETY: writing through a packed place writes the same memory because + // the field offset is zero, just with a lower alignment. + unsafe { crate::intrinsics::write_field_via_move::, T, 0>(dst, src) } } /// Performs a volatile read of the value from `src` without moving it. @@ -2809,3 +2802,6 @@ pub macro addr_of($place:expr) { pub macro addr_of_mut($place:expr) { &raw mut $place } + +#[repr(C, packed)] +struct Packed(T); diff --git a/tests/codegen-llvm/read_write_unaligned.rs b/tests/codegen-llvm/read_write_unaligned.rs new file mode 100644 index 0000000000000..021cb86c4d51b --- /dev/null +++ b/tests/codegen-llvm/read_write_unaligned.rs @@ -0,0 +1,53 @@ +//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes +//@ only-64bit + +#![crate_type = "lib"] + +use std::num::NonZero; +use std::ptr::NonNull; + +// CHECK-LABEL:nonnull ptr @read_unaligned_ptr(ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_ptr(ptr: *const NonNull) -> NonNull { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load ptr, ptr %ptr, align 1 + // CHECK-SAME: !nonnull + // CHECK-SAME: !noundef + // CHECK-NEXT: ret ptr [[TEMP]] + ptr.read_unaligned() +} + +// CHECK-LABEL:range(i16 1, 0) i16 @read_unaligned_i16(ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_i16(ptr: *const NonZero) -> NonZero { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load i16, ptr %ptr, align 1 + // CHECK-SAME: !range ![[R16:[0-9]+]] + // CHECK-SAME: !noundef + // CHECK-NEXT: ret i16 [[TEMP]] + ptr.read_unaligned() +} + +// CHECK-LABEL: void @typed_copy_unaligned_i32(ptr{{.+}}%src, ptr{{.+}}%dst) +#[no_mangle] +unsafe fn typed_copy_unaligned_i32(src: *const NonZero, dst: *mut NonZero) { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load i32, ptr %src, align 1 + // CHECK-SAME: !range ![[R32:[0-9]+]] + // CHECK-SAME: !noundef + // CHECK-NEXT: store i32 [[TEMP]], ptr %dst, align 1 + // CHECK-NEXT: ret void + dst.write_unaligned(src.read_unaligned()) +} + +// CHECK-LABEL: void @write_unaligned_i64(ptr{{.+}}%ptr, i64{{.+}}%val) +#[no_mangle] +unsafe fn write_unaligned_i64(ptr: *mut NonZero, val: NonZero) { + // CHECK: start: + // CHECK-NEXT: store i64 %val, ptr %ptr, align 1 + // CHECK-NEXT: ret void + ptr.write_unaligned(val) +} + +// CHECK: ![[R16]] = !{i16 1, i16 0} +// CHECK: ![[R32]] = !{i32 1, i32 0} diff --git a/tests/mir-opt/building/write_via_move.my_write_unaligned.CleanupPostBorrowck.after.mir b/tests/mir-opt/building/write_via_move.my_write_unaligned.CleanupPostBorrowck.after.mir new file mode 100644 index 0000000000000..bcbe889514255 --- /dev/null +++ b/tests/mir-opt/building/write_via_move.my_write_unaligned.CleanupPostBorrowck.after.mir @@ -0,0 +1,24 @@ +// MIR for `my_write_unaligned` after CleanupPostBorrowck + +fn my_write_unaligned(_1: *mut Foo, _2: String) -> () { + debug dst => _1; + debug src => _2; + let mut _0: (); + let mut _3: *mut Foo; + + bb0: { + StorageLive(_3); + _3 = copy _1; + ((*_3).1: std::string::String) = move _2; + StorageDead(_3); + drop(_2) -> [return: bb1, unwind: bb2]; + } + + bb1: { + return; + } + + bb2 (cleanup): { + resume; + } +} diff --git a/tests/mir-opt/building/write_via_move.rs b/tests/mir-opt/building/write_via_move.rs index 12be592a855c1..60aa5b67c4fec 100644 --- a/tests/mir-opt/building/write_via_move.rs +++ b/tests/mir-opt/building/write_via_move.rs @@ -18,6 +18,20 @@ fn box_new(x: T) -> Box<[T; 1024]> { unsafe { b.assume_init() } } +struct Foo(i32, String, u32); + +// EMIT_MIR write_via_move.my_write_unaligned.CleanupPostBorrowck.after.mir +// CHECK-LABEL: fn my_write_unaligned +#[inline(never)] +unsafe fn my_write_unaligned(dst: *mut Foo, src: String) { + // CHECK: [[TEMP:_.+]] = copy _1; + // CHECK: ((*[[TEMP]]).1: std::string::String) = move _2; + unsafe { std::intrinsics::write_field_via_move::<_, _, 1>(dst, src) } +} + fn main() { box_new(0); + + let mut foo = Foo(0, String::new(), 0); + unsafe { my_write_unaligned(&raw mut foo, String::new()) }; } diff --git a/tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-abort.diff new file mode 100644 index 0000000000000..c4fb3c4292307 --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-abort.diff @@ -0,0 +1,22 @@ +- // MIR for `read_field_via_copy_droppy` before LowerIntrinsics ++ // MIR for `read_field_via_copy_droppy` after LowerIntrinsics + + fn read_field_via_copy_droppy(_1: &Composite) -> String { + debug r => _1; + let mut _0: std::string::String; + let mut _2: *const Composite; + + bb0: { + StorageLive(_2); + _2 = &raw const (*_1); +- _0 = read_field_via_copy::(move _2) -> [return: bb1, unwind unreachable]; ++ _0 = copy ((*_2).1: std::string::String); ++ goto -> bb1; + } + + bb1: { + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-unwind.diff new file mode 100644 index 0000000000000..c4fb3c4292307 --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-unwind.diff @@ -0,0 +1,22 @@ +- // MIR for `read_field_via_copy_droppy` before LowerIntrinsics ++ // MIR for `read_field_via_copy_droppy` after LowerIntrinsics + + fn read_field_via_copy_droppy(_1: &Composite) -> String { + debug r => _1; + let mut _0: std::string::String; + let mut _2: *const Composite; + + bb0: { + StorageLive(_2); + _2 = &raw const (*_1); +- _0 = read_field_via_copy::(move _2) -> [return: bb1, unwind unreachable]; ++ _0 = copy ((*_2).1: std::string::String); ++ goto -> bb1; + } + + bb1: { + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/lower_intrinsics.rs b/tests/mir-opt/lower_intrinsics.rs index e8a7701477415..c0089ad4133fa 100644 --- a/tests/mir-opt/lower_intrinsics.rs +++ b/tests/mir-opt/lower_intrinsics.rs @@ -199,6 +199,19 @@ pub fn read_via_copy_uninhabited(r: &Never) -> Never { pub enum Never {} +// EMIT_MIR lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.diff +pub fn read_field_via_copy_droppy(r: &Composite) -> String { + // CHECK-LABEL: fn read_field_via_copy_droppy( + // CHECK: [[tmp:_.*]] = &raw const (*_1); + // CHECK: _0 = copy ((*[[tmp]]).1: std::string::String); + // CHECK-NOT: drop + // CHECK: return; + + unsafe { core::intrinsics::read_field_via_copy::(r) } +} + +pub struct Composite(i32, String, f32); + // EMIT_MIR lower_intrinsics.ptr_offset.LowerIntrinsics.diff pub unsafe fn ptr_offset(p: *const i32, d: isize) -> *const i32 { // CHECK-LABEL: fn ptr_offset( diff --git a/tests/mir-opt/pre-codegen/unaligned.rs b/tests/mir-opt/pre-codegen/unaligned.rs new file mode 100644 index 0000000000000..cced96d3e375c --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.rs @@ -0,0 +1,20 @@ +//@ compile-flags: -O -Zmir-opt-level=2 + +#![crate_type = "lib"] + +// EMIT_MIR unaligned.unaligned_typed_copy.runtime-optimized.after.mir +pub unsafe fn unaligned_typed_copy(src: *const String, dst: *mut String) { + // CHECK-LABEL: fn unaligned_typed_copy( + // CHECK: debug src => _1; + // CHECK: debug dst => _2; + // CHECK: debug val => [[VAL:_.+]]; + // CHECK: [[SRCU:_.+]] = copy _1 as *const std::ptr::Packed (PtrToPtr); + // CHECK: [[VAL]] = copy ((*[[SRCU]]).0: std::string::String); + // CHECK: [[DSTU:_.+]] = copy _2 as *mut std::ptr::Packed (PtrToPtr); + // CHECK: ((*[[DSTU]]).0: std::string::String) = copy [[VAL]]; + // CHECK-NOT: drop + unsafe { + let val = std::ptr::read_unaligned(src); + std::ptr::write_unaligned(dst, val); + } +} diff --git a/tests/mir-opt/pre-codegen/unaligned.unaligned_typed_copy.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/unaligned.unaligned_typed_copy.runtime-optimized.after.mir new file mode 100644 index 0000000000000..c64b87f1723e6 --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.unaligned_typed_copy.runtime-optimized.after.mir @@ -0,0 +1,39 @@ +// MIR for `unaligned_typed_copy` after runtime-optimized + +fn unaligned_typed_copy(_1: *const String, _2: *mut String) -> () { + debug src => _1; + debug dst => _2; + let mut _0: (); + let _4: std::string::String; + scope 1 { + debug val => _4; + scope 5 (inlined #[track_caller] write_unaligned::) { + let _5: *mut std::ptr::Packed; + scope 6 { + } + scope 7 (inlined std::ptr::mut_ptr::::cast::>) { + } + } + } + scope 2 (inlined #[track_caller] read_unaligned::) { + let _3: *const std::ptr::Packed; + scope 3 { + } + scope 4 (inlined std::ptr::const_ptr::::cast::>) { + } + } + + bb0: { + StorageLive(_4); + StorageLive(_3); + _3 = copy _1 as *const std::ptr::Packed (PtrToPtr); + _4 = copy ((*_3).0: std::string::String); + StorageDead(_3); + StorageLive(_5); + _5 = copy _2 as *mut std::ptr::Packed (PtrToPtr); + ((*_5).0: std::string::String) = copy _4; + StorageDead(_5); + StorageDead(_4); + return; + } +}