Preserve metadata like !range and !nonnull in unaligned operations

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -603,9 +603,13 @@ pub(crate) fn check_intrinsic_type(
         sym::cold_path => (0, 0, vec![], tcx.types.unit),
 
         sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
+        sym::read_field_via_copy => (2, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(1)),
         sym::write_via_move => {
             (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
         }
+        sym::write_field_via_move => {
+            (2, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(1)], tcx.types.unit)
+        }
         sym::write_box_via_move => {
             let t = param(0);
             let maybe_uninit_t = Ty::new_maybe_uninit(tcx, t);

compiler/rustc_mir_build/src/builder/expr/into.rs

@@ -401,14 +401,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::Call { ty, fun, ref args, .. }
                 if let ty::FnDef(def_id, generic_args) = *ty.kind()
                     && let Some(intrinsic) = this.tcx.intrinsic(def_id)
-                    && matches!(intrinsic.name, sym::write_via_move | sym::write_box_via_move) =>
+                    && matches!(
+                        intrinsic.name,
+                        sym::write_via_move | sym::write_field_via_move | sym::write_box_via_move
+                    ) =>
             {
                 // We still have to evaluate the callee expression as normal (but we don't care
                 // about its result).
                 let _fun = unpack!(block = this.as_local_operand(block, fun));
 
                 match intrinsic.name {
-                    sym::write_via_move => {
+                    sym::write_via_move | sym::write_field_via_move => {
                         // `write_via_move(ptr, val)` becomes `*ptr = val` but without any dropping.
 
                         // The destination must have unit type (so we don't actually have to store anything
@@ -424,7 +427,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             span_bug!(expr_span, "invalid write_via_move call")
                         };
                         let ptr_deref = ptr.project_deeper(&[ProjectionElem::Deref], this.tcx);
-                        this.expr_into_dest(ptr_deref, block, val)
+                        let dst_place = if intrinsic.name == sym::write_field_via_move {
+                            let field_ty = generic_args[1].expect_ty();
+                            let field_idx = generic_args[2].expect_const();
+                            let field_idx = field_idx.try_to_leaf().unwrap().to_u32();
+                            ptr_deref.project_deeper(
+                                &[PlaceElem::Field(FieldIdx::from_u32(field_idx), field_ty)],
+                                this.tcx,
+                            )
+                        } else {
+                            ptr_deref
+                        };
+                        this.expr_into_dest(dst_place, block, val)
                     }
                     sym::write_box_via_move => {
                         // The signature is:

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -1,5 +1,6 @@
 //! Lowers intrinsic calls
 
+use rustc_abi::FieldIdx;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_middle::{bug, span_bug};
@@ -148,7 +149,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         ));
                         terminator.kind = TerminatorKind::Goto { target };
                     }
-                    sym::read_via_copy => {
+                    sym::read_via_copy | sym::read_field_via_copy => {
                         let Ok([arg]) = take_array(args) else {
                             span_bug!(terminator.source_info.span, "Wrong number of arguments");
                         };
@@ -162,6 +163,17 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                                 "Only passing a local is supported"
                             );
                         };
+                        let derefed_place = if intrinsic.name == sym::read_field_via_copy {
+                            let field_ty = generic_args[1].expect_ty();
+                            let field_idx = generic_args[2].expect_const();
+                            let field_idx = field_idx.try_to_leaf().unwrap().to_u32();
+                            derefed_place.project_deeper(
+                                &[PlaceElem::Field(FieldIdx::from_u32(field_idx), field_ty)],
+                                tcx,
+                            )
+                        } else {
+                            derefed_place
+                        };
                         // Add new statement at the end of the block that does the read, and patch
                         // up the terminator.
                         block.statements.push(Statement::new(

compiler/rustc_span/src/symbol.rs

@@ -1649,6 +1649,7 @@ symbols! {
         raw_identifiers,
         raw_ref_op,
         re_rebalance_coherence,
+        read_field_via_copy,
         read_via_copy,
         readonly,
         realloc,
@@ -2348,6 +2349,7 @@ symbols! {
         wreg,
         write_box_via_move,
         write_bytes,
+        write_field_via_move,
         write_fmt,
         write_macro,
         write_str,

library/core/src/intrinsics/mod.rs

@@ -2202,7 +2202,7 @@ pub const fn carryless_mul<T: [const] fallback::CarrylessMul>(a: T, b: T) -> T {
     a.carryless_mul(b)
 }
 
-/// This is an implementation detail of [`crate::ptr::read`] and should
+/// This is an implementation detail of [`ptr::read`] and should
 /// not be used anywhere else.  See its comments for why this exists.
 ///
 /// This intrinsic can *only* be called where the pointer is a local without
@@ -2213,7 +2213,7 @@ pub const fn carryless_mul<T: [const] fallback::CarrylessMul>(a: T, b: T) -> T {
 #[rustc_intrinsic]
 pub const unsafe fn read_via_copy<T>(ptr: *const T) -> T;
 
-/// This is an implementation detail of [`crate::ptr::write`] and should
+/// This is an implementation detail of [`ptr::write`] and should
 /// not be used anywhere else.  See its comments for why this exists.
 ///
 /// This intrinsic can *only* be called where the pointer is a local without
@@ -2224,6 +2224,24 @@ pub const unsafe fn read_via_copy<T>(ptr: *const T) -> T;
 #[rustc_intrinsic]
 pub const unsafe fn write_via_move<T>(ptr: *mut T, value: T);
 
+/// Lowers to `RET = copy (*ptr)._FIELD_IDX` in MIR.
+///
+/// This is currently used only for [`ptr::read_unaligned`] so that it can emit
+/// a `packed` load, and shouldn't be used elsewhere.
+#[rustc_intrinsic_const_stable_indirect]
+#[rustc_nounwind]
+#[rustc_intrinsic]
+pub const unsafe fn read_field_via_copy<T: ?Sized, F, const FIELD_IDX: u32>(ptr: *const T) -> F;
+
+/// Lowers to `(*ptr)._FIELD_IDX = move value` in MIR.
+///
+/// This is currently used only for [`ptr::write_unaligned`] so that it can emit
+/// a `packed` store, and shouldn't be used elsewhere.
+#[rustc_intrinsic_const_stable_indirect]
+#[rustc_nounwind]
+#[rustc_intrinsic]
+pub const unsafe fn write_field_via_move<T: ?Sized, F, const FIELD_IDX: u32>(ptr: *mut T, value: F);
+
 /// Returns the value of the discriminant for the variant in 'v';
 /// if `T` has no discriminant, returns `0`.
 ///

library/core/src/ptr/mod.rs

@@ -1810,17 +1810,12 @@ pub const unsafe fn read<T>(src: *const T) -> T {
 #[track_caller]
 #[rustc_diagnostic_item = "ptr_read_unaligned"]
 pub const unsafe fn read_unaligned<T>(src: *const T) -> T {
-    let mut tmp = MaybeUninit::<T>::uninit();
-    // SAFETY: the caller must guarantee that `src` is valid for reads.
-    // `src` cannot overlap `tmp` because `tmp` was just allocated on
-    // the stack as a separate allocation.
-    //
-    // Also, since we just wrote a valid value into `tmp`, it is guaranteed
-    // to be properly initialized.
-    unsafe {
-        copy_nonoverlapping(src as *const u8, tmp.as_mut_ptr() as *mut u8, size_of::<T>());
-        tmp.assume_init()
-    }
+    let src = src.cast::<Packed<T>>();
+    // Always true because it's `repr(C, packed)`
+    const { assert!(mem::offset_of!(Packed<T>, 0) == 0) };
+    // SAFETY: reading through a packed place reads the same memory because
+    // the field offset is zero, just with a lower alignment.
+    unsafe { crate::intrinsics::read_field_via_copy::<Packed<T>, T, 0>(src) }
 }
 
 /// Overwrites a memory location with the given value without reading or
@@ -2012,14 +2007,12 @@ pub const unsafe fn write<T>(dst: *mut T, src: T) {
 #[rustc_diagnostic_item = "ptr_write_unaligned"]
 #[track_caller]
 pub const unsafe fn write_unaligned<T>(dst: *mut T, src: T) {
-    // SAFETY: the caller must guarantee that `dst` is valid for writes.
-    // `dst` cannot overlap `src` because the caller has mutable access
-    // to `dst` while `src` is owned by this function.
-    unsafe {
-        copy_nonoverlapping((&raw const src) as *const u8, dst as *mut u8, size_of::<T>());
-        // We are calling the intrinsic directly to avoid function calls in the generated code.
-        intrinsics::forget(src);
-    }
+    let dst = dst.cast::<Packed<T>>();
+    // Always true because it's `repr(C, packed)`
+    const { assert!(mem::offset_of!(Packed<T>, 0) == 0) };
+    // SAFETY: writing through a packed place writes the same memory because
+    // the field offset is zero, just with a lower alignment.
+    unsafe { crate::intrinsics::write_field_via_move::<Packed<T>, T, 0>(dst, src) }
 }
 
 /// Performs a volatile read of the value from `src` without moving it.
@@ -2809,3 +2802,6 @@ pub macro addr_of($place:expr) {
 pub macro addr_of_mut($place:expr) {
     &raw mut $place
 }
+
+#[repr(C, packed)]
+struct Packed<T>(T);

tests/codegen-llvm/read_write_unaligned.rs

@@ -0,0 +1,53 @@
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
+//@ only-64bit
+
+#![crate_type = "lib"]
+
+use std::num::NonZero;
+use std::ptr::NonNull;
+
+// CHECK-LABEL:nonnull ptr @read_unaligned_ptr(ptr{{.+}}%ptr)
+#[no_mangle]
+unsafe fn read_unaligned_ptr(ptr: *const NonNull<i16>) -> NonNull<i16> {
+    // CHECK: start:
+    // CHECK-NEXT: [[TEMP:%.+]] = load ptr, ptr %ptr, align 1
+    // CHECK-SAME: !nonnull
+    // CHECK-SAME: !noundef
+    // CHECK-NEXT: ret ptr [[TEMP]]
+    ptr.read_unaligned()
+}
+
+// CHECK-LABEL:range(i16 1, 0) i16 @read_unaligned_i16(ptr{{.+}}%ptr)
+#[no_mangle]
+unsafe fn read_unaligned_i16(ptr: *const NonZero<i16>) -> NonZero<i16> {
+    // CHECK: start:
+    // CHECK-NEXT: [[TEMP:%.+]] = load i16, ptr %ptr, align 1
+    // CHECK-SAME: !range ![[R16:[0-9]+]]
+    // CHECK-SAME: !noundef
+    // CHECK-NEXT: ret i16 [[TEMP]]
+    ptr.read_unaligned()
+}
+
+// CHECK-LABEL: void @typed_copy_unaligned_i32(ptr{{.+}}%src, ptr{{.+}}%dst)
+#[no_mangle]
+unsafe fn typed_copy_unaligned_i32(src: *const NonZero<i32>, dst: *mut NonZero<i32>) {
+    // CHECK: start:
+    // CHECK-NEXT: [[TEMP:%.+]] = load i32, ptr %src, align 1
+    // CHECK-SAME: !range ![[R32:[0-9]+]]
+    // CHECK-SAME: !noundef
+    // CHECK-NEXT: store i32 [[TEMP]], ptr %dst, align 1
+    // CHECK-NEXT: ret void
+    dst.write_unaligned(src.read_unaligned())
+}
+
+// CHECK-LABEL: void @write_unaligned_i64(ptr{{.+}}%ptr, i64{{.+}}%val)
+#[no_mangle]
+unsafe fn write_unaligned_i64(ptr: *mut NonZero<i64>, val: NonZero<i64>) {
+    // CHECK: start:
+    // CHECK-NEXT: store i64 %val, ptr %ptr, align 1
+    // CHECK-NEXT: ret void
+    ptr.write_unaligned(val)
+}
+
+// CHECK: ![[R16]] = !{i16 1, i16 0}
+// CHECK: ![[R32]] = !{i32 1, i32 0}

tests/mir-opt/building/write_via_move.my_write_unaligned.CleanupPostBorrowck.after.mir

@@ -0,0 +1,24 @@
+// MIR for `my_write_unaligned` after CleanupPostBorrowck
+
+fn my_write_unaligned(_1: *mut Foo, _2: String) -> () {
+    debug dst => _1;
+    debug src => _2;
+    let mut _0: ();
+    let mut _3: *mut Foo;
+
+    bb0: {
+        StorageLive(_3);
+        _3 = copy _1;
+        ((*_3).1: std::string::String) = move _2;
+        StorageDead(_3);
+        drop(_2) -> [return: bb1, unwind: bb2];
+    }
+
+    bb1: {
+        return;
+    }
+
+    bb2 (cleanup): {
+        resume;
+    }
+}

tests/mir-opt/building/write_via_move.rs

@@ -18,6 +18,20 @@ fn box_new<T: Copy>(x: T) -> Box<[T; 1024]> {
     unsafe { b.assume_init() }
 }
 
+struct Foo(i32, String, u32);
+
+// EMIT_MIR write_via_move.my_write_unaligned.CleanupPostBorrowck.after.mir
+// CHECK-LABEL: fn my_write_unaligned
+#[inline(never)]
+unsafe fn my_write_unaligned(dst: *mut Foo, src: String) {
+    // CHECK: [[TEMP:_.+]] = copy _1;
+    // CHECK: ((*[[TEMP]]).1: std::string::String) = move _2;
+    unsafe { std::intrinsics::write_field_via_move::<_, _, 1>(dst, src) }
+}
+
 fn main() {
     box_new(0);
+
+    let mut foo = Foo(0, String::new(), 0);
+    unsafe { my_write_unaligned(&raw mut foo, String::new()) };
 }

tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-abort.diff

@@ -0,0 +1,22 @@
+- // MIR for `read_field_via_copy_droppy` before LowerIntrinsics
++ // MIR for `read_field_via_copy_droppy` after LowerIntrinsics
+
+  fn read_field_via_copy_droppy(_1: &Composite) -> String {
+      debug r => _1;
+      let mut _0: std::string::String;
+      let mut _2: *const Composite;
+
+      bb0: {
+          StorageLive(_2);
+          _2 = &raw const (*_1);
+-         _0 = read_field_via_copy::<Composite, String, 1>(move _2) -> [return: bb1, unwind unreachable];
++         _0 = copy ((*_2).1: std::string::String);
++         goto -> bb1;
+      }
+
+      bb1: {
+          StorageDead(_2);
+          return;
+      }
+  }
+

tests/mir-opt/lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.panic-unwind.diff

@@ -0,0 +1,22 @@
+- // MIR for `read_field_via_copy_droppy` before LowerIntrinsics
++ // MIR for `read_field_via_copy_droppy` after LowerIntrinsics
+
+  fn read_field_via_copy_droppy(_1: &Composite) -> String {
+      debug r => _1;
+      let mut _0: std::string::String;
+      let mut _2: *const Composite;
+
+      bb0: {
+          StorageLive(_2);
+          _2 = &raw const (*_1);
+-         _0 = read_field_via_copy::<Composite, String, 1>(move _2) -> [return: bb1, unwind unreachable];
++         _0 = copy ((*_2).1: std::string::String);
++         goto -> bb1;
+      }
+
+      bb1: {
+          StorageDead(_2);
+          return;
+      }
+  }
+

tests/mir-opt/lower_intrinsics.rs

@@ -199,6 +199,19 @@ pub fn read_via_copy_uninhabited(r: &Never) -> Never {
 
 pub enum Never {}
 
+// EMIT_MIR lower_intrinsics.read_field_via_copy_droppy.LowerIntrinsics.diff
+pub fn read_field_via_copy_droppy(r: &Composite) -> String {
+    // CHECK-LABEL: fn read_field_via_copy_droppy(
+    // CHECK: [[tmp:_.*]] = &raw const (*_1);
+    // CHECK: _0 = copy ((*[[tmp]]).1: std::string::String);
+    // CHECK-NOT: drop
+    // CHECK: return;
+
+    unsafe { core::intrinsics::read_field_via_copy::<Composite, String, 1>(r) }
+}
+
+pub struct Composite(i32, String, f32);
+
 // EMIT_MIR lower_intrinsics.ptr_offset.LowerIntrinsics.diff
 pub unsafe fn ptr_offset(p: *const i32, d: isize) -> *const i32 {
     // CHECK-LABEL: fn ptr_offset(

tests/mir-opt/pre-codegen/unaligned.rs

@@ -0,0 +1,20 @@
+//@ compile-flags: -O -Zmir-opt-level=2
+
+#![crate_type = "lib"]
+
+// EMIT_MIR unaligned.unaligned_typed_copy.runtime-optimized.after.mir
+pub unsafe fn unaligned_typed_copy(src: *const String, dst: *mut String) {
+    // CHECK-LABEL: fn unaligned_typed_copy(
+    // CHECK: debug src => _1;
+    // CHECK: debug dst => _2;
+    // CHECK: debug val => [[VAL:_.+]];
+    // CHECK: [[SRCU:_.+]] = copy _1 as *const std::ptr::Packed<std::string::String> (PtrToPtr);
+    // CHECK: [[VAL]] = copy ((*[[SRCU]]).0: std::string::String);
+    // CHECK: [[DSTU:_.+]] = copy _2 as *mut std::ptr::Packed<std::string::String> (PtrToPtr);
+    // CHECK: ((*[[DSTU]]).0: std::string::String) = copy [[VAL]];
+    // CHECK-NOT: drop
+    unsafe {
+        let val = std::ptr::read_unaligned(src);
+        std::ptr::write_unaligned(dst, val);
+    }
+}