From d8e5a3d551870aa8c30eab724aaf72fad81df563 Mon Sep 17 00:00:00 2001 From: Cathal Mullan Date: Sat, 30 May 2026 01:23:56 +0100 Subject: [PATCH 01/51] Implement aarch64 SHA-1/SHA-512 LLVM intrinsics --- example/neon.rs | 130 ++++++++++++++++ src/intrinsics/llvm_aarch64.rs | 265 +++++++++++++++++++++++++++++++++ 2 files changed, 395 insertions(+) diff --git a/example/neon.rs b/example/neon.rs index 1aec5badcbc26..6b024de7bb560 100644 --- a/example/neon.rs +++ b/example/neon.rs @@ -307,6 +307,75 @@ unsafe fn test_vaesimcq_u8() { assert_eq!(r, e); } +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha1cq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1c + let a = u32x4::from([0, 1, 2, 3]); + let b = 8; + let c = u32x4::from([4, 5, 6, 7]); + let e = u32x4::from([0x40072911, 0x40003948, 0x80000072, 0x80000003]); + let r: u32x4 = unsafe { transmute(vsha1cq_u32(transmute(a), b, transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +fn test_vsha1h_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1h + let a = 8; + let e = 0x00000002; + let r = vsha1h_u32(a); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha1mq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1m + let a = u32x4::from([0, 1, 2, 3]); + let b = 8; + let c = u32x4::from([4, 5, 6, 7]); + let e = u32x4::from([0x4007a107, 0x00003d08, 0x0000007a, 0xc0000003]); + let r: u32x4 = unsafe { transmute(vsha1mq_u32(transmute(a), b, transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha1pq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1p + let a = u32x4::from([0, 1, 2, 3]); + let b = 8; + let c = u32x4::from([4, 5, 6, 7]); + let e = u32x4::from([0x80062d18, 0x4000315c, 0x90000062, 0x00000003]); + let r: u32x4 = unsafe { transmute(vsha1pq_u32(transmute(a), b, transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha1su0q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1su0 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([0x0000000a, 0x0000000b, 0x0000000c, 0x0000000d]); + let r: u32x4 = unsafe { transmute(vsha1su0q_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha1su1q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha1su1 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let e = u32x4::from([0x0000000a, 0x0000000e, 0x0000000a, 0x00000012]); + let r: u32x4 = unsafe { transmute(vsha1su1q_u32(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + #[cfg(target_arch = "aarch64")] #[target_feature(enable = "sha2")] unsafe fn test_vsha256hq_u32() { @@ -354,6 +423,53 @@ unsafe fn test_vsha256su1q_u32() { assert_eq!(r, e); } +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vsha512hq_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512h + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let c = u64x2::from([4, 5]); + let e = u64x2::from([0x001c805053800005, 0x0015400002800003]); + let r: u64x2 = unsafe { transmute(vsha512hq_u64(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vsha512h2q_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512h2 + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let c = u64x2::from([4, 5]); + let e = u64x2::from([0x401000514a000405, 0x0000004108000005]); + let r: u64x2 = unsafe { transmute(vsha512h2q_u64(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vsha512su0q_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512su0 + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let e = u64x2::from([0x8100000000000000, 0x0200000000000002]); + let r: u64x2 = unsafe { transmute(vsha512su0q_u64(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vsha512su1q_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha512su1 + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let c = u64x2::from([4, 5]); + let e = u64x2::from([0x0000400000000014, 0x000060000000001e]); + let r: u64x2 = unsafe { transmute(vsha512su1q_u64(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes")] fn test_vmull_p64() { @@ -565,11 +681,25 @@ fn main() { test_vaesmcq_u8(); test_vaesimcq_u8(); + test_vsha1cq_u32(); + test_vsha1h_u32(); + test_vsha1mq_u32(); + test_vsha1pq_u32(); + test_vsha1su0q_u32(); + test_vsha1su1q_u32(); + test_vsha256hq_u32(); test_vsha256h2q_u32(); test_vsha256su0q_u32(); test_vsha256su1q_u32(); + if std::arch::is_aarch64_feature_detected!("sha3") { + test_vsha512hq_u64(); + test_vsha512h2q_u64(); + test_vsha512su0q_u64(); + test_vsha512su1q_u64(); + } + test_vmull_p64(); test_vmull_p8(); diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index 9da87a5774e8f..c322859fec2b3 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -605,6 +605,164 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( ); } + "llvm.aarch64.crypto.sha1c" | "llvm.aarch64.crypto.sha1m" | "llvm.aarch64.crypto.sha1p" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.sha1c" => { + "fmov s2, w1 + sha1c q0, s2, v1.4s" + } + "llvm.aarch64.crypto.sha1m" => { + "fmov s2, w1 + sha1m q0, s2, v1.4s" + } + "llvm.aarch64.crypto.sha1p" => { + "fmov s2, w1 + sha1p q0, s2, v1.4s" + } + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: c, + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + late: true, + place: None, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha1h" => { + intrinsic_args!(fx, args => (a); intrinsic); + + let a = a.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + "fmov s0, w0 + sha1h s0, s0 + fmov w0, s0" + .into(), + )], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + late: true, + place: None, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha1su0" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha1su0 v0.4s, v1.4s, v2.4s".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + value: c, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha1su1" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha1su1 v0.4s, v1.4s".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + "llvm.aarch64.crypto.sha256h" | "llvm.aarch64.crypto.sha256h2" => { intrinsic_args!(fx, args => (a, b, c); intrinsic); @@ -712,6 +870,113 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( ); } + "llvm.aarch64.crypto.sha512h" | "llvm.aarch64.crypto.sha512h2" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.sha512h" => "sha512h q0, q1, v2.2d", + "llvm.aarch64.crypto.sha512h2" => "sha512h2 q0, q1, v2.2d", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + value: c, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha512su0" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha512su0 v0.2d, v1.2d".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha512su1" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha512su1 v0.2d, v1.2d, v2.2d".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + value: c, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + "llvm.aarch64.neon.pmull64" => { intrinsic_args!(fx, args => (a, b); intrinsic); From e945563dad217fffde0cbc665066c6a340032694 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 27 May 2026 12:22:03 +0200 Subject: [PATCH 02/51] add `extern "tail"` calling convention --- src/abi/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index a8b179f169bf6..9644932ae1055 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -55,8 +55,9 @@ pub(crate) fn conv_to_call_conv( match c { CanonAbi::Rust | CanonAbi::RustCold | CanonAbi::C => default_call_conv, - // Cranelift doesn't currently have anything for this. - CanonAbi::RustPreserveNone => default_call_conv, + CanonAbi::RustPreserveNone | CanonAbi::RustTail => { + sess.dcx().fatal(format!("call conv {c:?} is LLVM-specific")) + } // Functions with this calling convention can only be called from assembly, but it is // possible to declare an `extern "custom"` block, so the backend still needs a calling @@ -71,7 +72,7 @@ pub(crate) fn conv_to_call_conv( }, CanonAbi::Interrupt(_) | CanonAbi::Arm(_) | CanonAbi::Swift => { - sess.dcx().fatal("call conv {c:?} is not yet implemented") + sess.dcx().fatal(format!("call conv {c:?} is not yet implemented")) } CanonAbi::GpuKernel => { unreachable!("tried to use {c:?} call conv which only exists on an unsupported target") From ce5a62c20698c9fbf9c4e3660a3d021ae53e1d8f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 6 Jun 2026 16:27:31 +0200 Subject: [PATCH 03/51] Merge commit 'f2ef4150da1c50e5b04607cdab771530cac4ffe4' into sync_cg_clif-2026-06-06 --- .../workflows/cranelift-release-branch.yml | 64 ++ .github/workflows/main.yml | 11 +- Cargo.lock | 105 ++-- Cargo.toml | 26 +- build_system/abi_cafe.rs | 1 + build_system/bench.rs | 1 + build_system/build_sysroot.rs | 2 + build_system/main.rs | 1 - build_system/prepare.rs | 21 +- build_system/tests.rs | 50 ++ config.txt | 3 +- example/neon.rs | 319 ++++++++++ example/std_example.rs | 8 +- ...d-Disable-f16-usage-in-portable-simd.patch | 15 + ...0027-stdlib-128bit-atomic-operations.patch | 8 +- .../0029-sysroot_tests-disable-f16-math.patch | 480 +++++++++++++- rust-toolchain.toml | 2 +- scripts/setup_rust_fork.sh | 2 +- scripts/test_rustc_tests.sh | 3 + src/abi/mod.rs | 61 +- src/abi/pass_mode.rs | 41 +- src/allocator.rs | 11 +- src/analyze.rs | 12 +- src/base.rs | 11 +- src/concurrency_limiter.rs | 205 ------ src/constant.rs | 3 +- src/debuginfo/unwind.rs | 2 +- src/driver/aot.rs | 595 +++++++----------- src/driver/jit.rs | 5 +- src/global_asm.rs | 25 +- src/inline_asm.rs | 18 +- src/intrinsics/llvm_aarch64.rs | 379 +++++++++++ src/intrinsics/simd.rs | 2 +- src/lib.rs | 26 +- src/value_and_place.rs | 15 +- 35 files changed, 1764 insertions(+), 769 deletions(-) create mode 100644 .github/workflows/cranelift-release-branch.yml delete mode 100644 src/concurrency_limiter.rs diff --git a/.github/workflows/cranelift-release-branch.yml b/.github/workflows/cranelift-release-branch.yml new file mode 100644 index 0000000000000..5cfdc3e8f2936 --- /dev/null +++ b/.github/workflows/cranelift-release-branch.yml @@ -0,0 +1,64 @@ +name: Test upcoming Cranelift release branch + +on: + schedule: + - cron: "0 3 6 * *" + workflow_dispatch: {} + +permissions: {} + +env: + CARGO_BUILD_INCREMENTAL: false + RUSTFLAGS: "-Dwarnings" + +jobs: + test_upcoming_cranelift_release: + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - uses: actions/checkout@v6 + + - name: Determine latest Wasmtime release branch + id: wasmtime_release_branch + run: | + branches="$( + git ls-remote --heads https://github.com/bytecodealliance/wasmtime.git "refs/heads/release-*" \ + | awk '{print $2}' \ + | sed 's#refs/heads/##' \ + | sort -V + )" + if [[ -z "${branches}" ]]; then + echo "No wasmtime release branches found" + exit 1 + fi + latest="$(echo "${branches}" | tail -n 1)" + echo "Latest release branch: ${latest}" + echo "branch=${latest}" >> "$GITHUB_OUTPUT" + + - name: Patch Cargo.toml to use release branch Cranelift + run: | + cat >>Cargo.toml <", ); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 216c87f095533..5c8b719ad5404 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -231,6 +231,8 @@ fn build_clif_sysroot_for_triple( // inlining. rustflags.push("-Zinline-mir".to_owned()); + rustflags.push("-Zdisable-incr-comp-backend-caching".to_owned()); + if let Some(prefix) = env::var_os("CG_CLIF_STDLIB_REMAP_PATH_PREFIX") { rustflags.push("--remap-path-prefix".to_owned()); rustflags.push(format!("library/={}/library", prefix.to_str().unwrap())); diff --git a/build_system/main.rs b/build_system/main.rs index 0720d72c6d7cb..852fda950d88b 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -59,7 +59,6 @@ fn main() { if env::var_os("RUST_BACKTRACE").is_none() { env::set_var("RUST_BACKTRACE", "1"); } - env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); let mut args = env::args().skip(1); let command = match args.next().as_deref() { diff --git a/build_system/prepare.rs b/build_system/prepare.rs index ba5cc9a29f599..1bc56e311ec7d 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -11,11 +11,13 @@ pub(crate) fn prepare(dirs: &Dirs) { std::fs::create_dir_all(&dirs.download_dir).unwrap(); crate::tests::RAND_REPO.fetch(dirs); crate::tests::REGEX_REPO.fetch(dirs); + crate::tests::GRAVIOLA_REPO.fetch(dirs); } pub(crate) struct GitRepo { url: GitRepoUrl, rev: &'static str, + submodules: &'static [&'static str], content_hash: &'static str, patch_name: &'static str, } @@ -71,10 +73,17 @@ impl GitRepo { user: &'static str, repo: &'static str, rev: &'static str, + submodules: &'static [&'static str], content_hash: &'static str, patch_name: &'static str, ) -> GitRepo { - GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name } + GitRepo { + url: GitRepoUrl::Github { user, repo }, + rev, + submodules, + content_hash, + patch_name, + } } fn download_dir(&self, dirs: &Dirs) -> PathBuf { @@ -132,6 +141,7 @@ impl GitRepo { &download_dir, &format!("https://github.com/{}/{}.git", user, repo), self.rev, + self.submodules, ); } } @@ -160,7 +170,7 @@ impl GitRepo { } } -fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { +fn clone_repo(download_dir: &Path, repo: &str, rev: &str, submodules: &[&str]) { eprintln!("[CLONE] {}", repo); match fs::remove_dir_all(download_dir) { @@ -180,6 +190,13 @@ fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { checkout_cmd.arg("-q").arg(rev); spawn_and_wait(checkout_cmd); + if !submodules.is_empty() { + let mut submodule_cmd = git_command(download_dir, "submodule"); + submodule_cmd.arg("update").arg("--init"); + submodule_cmd.args(submodules); + spawn_and_wait(submodule_cmd); + } + std::fs::remove_dir_all(download_dir.join(".git")).unwrap(); } diff --git a/build_system/tests.rs b/build_system/tests.rs index 3b6a2e7a055cb..685bf8ce9a891 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -125,6 +125,7 @@ pub(crate) static RAND_REPO: GitRepo = GitRepo::github( "rust-random", "rand", "1f4507a8e1cf8050e4ceef95eeda8f64645b6719", + &[], "981f8bf489338978", "rand", ); @@ -135,12 +136,24 @@ pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( "rust-lang", "regex", "061ee815ef2c44101dba7b0b124600fcb03c1912", + &[], "dc26aefbeeac03ca", "regex", ); static REGEX: CargoProject = CargoProject::new(REGEX_REPO.source_dir(), "regex_target"); +pub(crate) static GRAVIOLA_REPO: GitRepo = GitRepo::github( + "ctz", + "graviola", + "c779b83cfd7114c4802293700c92cfb5e05cb4b7", + &["thirdparty/cavp", "thirdparty/wycheproof"], + "e0925ceb21a56101", + "graviola", +); + +static GRAVIOLA: CargoProject = CargoProject::new(GRAVIOLA_REPO.source_dir(), "graviola_target"); + static PORTABLE_SIMD_SRC: RelPath = RelPath::build("portable-simd"); static PORTABLE_SIMD: CargoProject = CargoProject::new(PORTABLE_SIMD_SRC, "portable-simd_target"); @@ -199,6 +212,43 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ spawn_and_wait(build_cmd); } }), + TestCase::custom("test.graviola", &|runner| { + let (arch, _) = runner.target_compiler.triple.split_once('-').unwrap(); + + if !["aarch64", "x86_64"].contains(&arch) { + eprintln!("Skipping `graviola` tests: unsupported target"); + return; + } + + GRAVIOLA_REPO.patch(&runner.dirs); + GRAVIOLA.clean(&runner.dirs); + + if runner.is_native { + let mut test_cmd = GRAVIOLA.test(&runner.target_compiler, &runner.dirs); + + // FIXME: Disable AVX-512 until intrinsics are supported. + test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512f", "1"); + test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512bw", "1"); + test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512vl", "1"); + + test_cmd.args([ + "-p", + "graviola", + "--lib", + "--", + "-q", + // FIXME: Disable AVX-512 until intrinsics are supported. + "--skip", + "check_counter512", + ]); + spawn_and_wait(test_cmd); + } else { + eprintln!("Cross-Compiling: Not running tests"); + let mut build_cmd = GRAVIOLA.build(&runner.target_compiler, &runner.dirs); + build_cmd.args(["-p", "graviola", "--lib"]); + spawn_and_wait(build_cmd); + } + }), TestCase::custom("test.portable-simd", &|runner| { apply_patches( &runner.dirs, diff --git a/config.txt b/config.txt index 72631355733c4..7c516e2164b40 100644 --- a/config.txt +++ b/config.txt @@ -20,7 +20,7 @@ aot.mini_core_hello_world testsuite.base_sysroot aot.arbitrary_self_types_pointers_and_wrappers -#jit.std_example # FIXME(#1619) broken for some reason +jit.std_example aot.std_example aot.dst_field_align aot.subslice-patterns-const-eval @@ -36,4 +36,5 @@ test.sysroot testsuite.extended_sysroot test.rust-random/rand test.regex +test.graviola test.portable-simd diff --git a/example/neon.rs b/example/neon.rs index 98a2a7af38f6b..1aec5badcbc26 100644 --- a/example/neon.rs +++ b/example/neon.rs @@ -9,6 +9,25 @@ use std::mem::transmute; #[cfg(target_arch = "aarch64")] use std::simd::*; +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "crc")] +unsafe fn test_crc32() { + assert!(std::arch::is_aarch64_feature_detected!("crc")); + + let a: u32 = 42; + let b: u64 = 0xdeadbeef; + + assert_eq!(__crc32b(a, b as u8), 0xEB0E363F); + assert_eq!(__crc32h(a, b as u16), 0x9A54BD80); + assert_eq!(__crc32w(a, b as u32), 0xF491F059); + assert_eq!(__crc32d(a, b as u64), 0xD14BBEA6); + + assert_eq!(__crc32cb(a, b as u8), 0xF67C32D8); + assert_eq!(__crc32ch(a, b as u16), 0x479108B8); + assert_eq!(__crc32cw(a, b as u32), 0x979F49F8); + assert_eq!(__crc32cd(a, b as u64), 0x0E6BE593); +} + #[cfg(target_arch = "aarch64")] unsafe fn test_vpmin_s8() { let a = i8x8::from([1, -2, 3, -4, 5, 6, 7, 8]); @@ -240,6 +259,272 @@ unsafe fn test_vrndnq_f32() { assert_eq!(r, e); } +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaeseq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aese + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let e = u8x16::from([ + 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, + 0xca, + ]); + let r: u8x16 = unsafe { transmute(vaeseq_u8(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaesdq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesd + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let e = u8x16::from([ + 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, + 0x7c, + ]); + let r: u8x16 = unsafe { transmute(vaesdq_u8(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaesmcq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesmc + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let e = u8x16::from([2, 7, 0, 5, 6, 3, 4, 1, 10, 15, 8, 13, 14, 11, 12, 9]); + let r: u8x16 = unsafe { transmute(vaesmcq_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaesimcq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesimc + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let e = u8x16::from([10, 15, 8, 13, 14, 11, 12, 9, 2, 7, 0, 5, 6, 3, 4, 1]); + let r: u8x16 = unsafe { transmute(vaesimcq_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256hq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256h + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([0x27bb4ae0, 0xd8f61f7c, 0xb7c1ecdc, 0x10800215]); + let r: u32x4 = unsafe { transmute(vsha256hq_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256h2q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256h2 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([0x6989ee0d, 0x4b055920, 0x52800a12, 0x00000014]); + let r: u32x4 = unsafe { transmute(vsha256h2q_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256su0q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256su0 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let e = u32x4::from([0x02004000, 0x04008001, 0x0600c002, 0x08010003]); + let r: u32x4 = unsafe { transmute(vsha256su0q_u32(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256su1q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256su1 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([0x00044005, 0x0004e007, 0xa802211b, 0xec036145]); + let r: u32x4 = unsafe { transmute(vsha256su1q_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +fn test_vmull_p64() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull64 + let a: u64 = 3; + let b: u64 = 6; + let e: u128 = 10; + let r: u128 = vmull_p64(a, b); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vmull_p8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull.v8i16 + let a = u8x8::from([0, 1, 2, 3, 4, 5, 6, 7]); + let b = u8x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let e = u16x8::from([0x0000, 0x0009, 0x0014, 0x001d, 0x0030, 0x0039, 0x0024, 0x002d]); + let r: u16x8 = unsafe { transmute(vmull_p8(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulh_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v4i16 + let a = i16x4::from([1, 2, 4, 8]); + let b = i16x4::from([16384, 16384, 16384, 16384]); + let e = i16x4::from([0, 1, 2, 4]); + let r: i16x4 = unsafe { transmute(vqdmulh_s16(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulh_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v2i32 + let a = i32x2::from([1, 2]); + let b = i32x2::from([1073741824, 1073741824]); + let e = i32x2::from([0, 1]); + let r: i32x2 = unsafe { transmute(vqdmulh_s32(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulhq_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v8i16 + let a = i16x8::from([1, 2, 4, 8, 16, 32, 64, 128]); + let b = i16x8::from([16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]); + let e = i16x8::from([0, 1, 2, 4, 8, 16, 32, 64]); + let r: i16x8 = unsafe { transmute(vqdmulhq_s16(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulhq_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v4i32 + let a = i32x4::from([1, 2, 4, 8]); + let b = i32x4::from([1073741824, 1073741824, 1073741824, 1073741824]); + let e = i32x4::from([0, 1, 2, 4]); + let r: i32x4 = unsafe { transmute(vqdmulhq_s32(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_s8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v4i16.v8i8 + let a = i8x8::from([1, 2, 3, 4, -5, -6, -7, -8]); + let e = i16x4::from([3, 7, -11, -15]); + let r: i16x4 = unsafe { transmute(vpaddl_s8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v2i32.v4i16 + let a = i16x4::from([1, 2, -3, -4]); + let e = i32x2::from([3, -7]); + let r: i32x2 = unsafe { transmute(vpaddl_s16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v1i64.v2i32 + let a = i32x2::from([1, -2]); + let e = i64x1::from([-1]); + let r: i64x1 = unsafe { transmute(vpaddl_s32(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_s8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v8i16.v16i8 + let a = i8x16::from([1, 2, 3, 4, 5, 6, 7, 8, -9, -10, -11, -12, -13, -14, -15, -16]); + let e = i16x8::from([3, 7, 11, 15, -19, -23, -27, -31]); + let r: i16x8 = unsafe { transmute(vpaddlq_s8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v4i32.v8i16 + let a = i16x8::from([1, 2, 3, 4, -5, -6, -7, -8]); + let e = i32x4::from([3, 7, -11, -15]); + let r: i32x4 = unsafe { transmute(vpaddlq_s16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v2i64.v4i32 + let a = i32x4::from([1, 2, -3, -4]); + let e = i64x2::from([3, -7]); + let r: i64x2 = unsafe { transmute(vpaddlq_s32(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v4i16.v8i8 + let a = u8x8::from([255, 254, 253, 252, 251, 250, 249, 248]); + let e = u16x4::from([509, 505, 501, 497]); + let r: u16x4 = unsafe { transmute(vpaddl_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_u16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v2i32.v4i16 + let a = u16x4::from([65535, 65534, 65533, 65532]); + let e = u32x2::from([131069, 131065]); + let r: u32x2 = unsafe { transmute(vpaddl_u16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v1i64.v2i32 + let a = u32x2::from([4294967295, 4294967294]); + let e = u64x1::from([8589934589]); + let r: u64x1 = unsafe { transmute(vpaddl_u32(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v8i16.v16i8 + let a = u8x16::from([ + 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, + ]); + let e = u16x8::from([509, 505, 501, 497, 493, 489, 485, 481]); + let r: u16x8 = unsafe { transmute(vpaddlq_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_u16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v4i32.v8i16 + let a = u16x8::from([65535, 65534, 65533, 65532, 65531, 65530, 65529, 65528]); + let e = u32x4::from([131069, 131065, 131061, 131057]); + let r: u32x4 = unsafe { transmute(vpaddlq_u16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v2i64.v4i32 + let a = u32x4::from([4294967295, 4294967294, 4294967293, 4294967292]); + let e = u64x2::from([8589934589, 8589934585]); + let r: u64x2 = unsafe { transmute(vpaddlq_u32(transmute(a))) }; + assert_eq!(r, e); +} + #[cfg(target_arch = "aarch64")] fn main() { unsafe { @@ -272,6 +557,40 @@ fn main() { test_vminq_f32(); test_vaddvq_f32(); test_vrndnq_f32(); + + test_crc32(); + + test_vaeseq_u8(); + test_vaesdq_u8(); + test_vaesmcq_u8(); + test_vaesimcq_u8(); + + test_vsha256hq_u32(); + test_vsha256h2q_u32(); + test_vsha256su0q_u32(); + test_vsha256su1q_u32(); + + test_vmull_p64(); + test_vmull_p8(); + + test_vqdmulh_s16(); + test_vqdmulh_s32(); + test_vqdmulhq_s16(); + test_vqdmulhq_s32(); + + test_vpaddl_s8(); + test_vpaddl_s16(); + test_vpaddl_s32(); + test_vpaddlq_s8(); + test_vpaddlq_s16(); + test_vpaddlq_s32(); + + test_vpaddl_u8(); + test_vpaddl_u16(); + test_vpaddl_u32(); + test_vpaddlq_u8(); + test_vpaddlq_u16(); + test_vpaddlq_u32(); } } diff --git a/example/std_example.rs b/example/std_example.rs index f0e38ae0610c9..252344b378e8a 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -1,8 +1,6 @@ #![feature(core_intrinsics, coroutines, coroutine_trait, repr_simd, tuple_trait, unboxed_closures)] #![allow(internal_features)] -#[cfg(target_arch = "x86_64")] -use std::arch::asm; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; use std::hint::black_box; @@ -591,7 +589,7 @@ unsafe fn test_xmm_roundtrip() { let input = [1u8; 16]; let mut output = [0u8; 16]; - asm!( + std::arch::asm!( "movups {xmm}, [{input}]", "movups [{output}], {xmm}", input = in(reg) input.as_ptr(), @@ -611,7 +609,7 @@ unsafe fn test_ymm_roundtrip() { let input = [1u8; 32]; let mut output = [0u8; 32]; - asm!( + std::arch::asm!( "vmovups {ymm}, [{input}]", "vmovups [{output}], {ymm}", input = in(reg) input.as_ptr(), @@ -631,7 +629,7 @@ unsafe fn test_zmm_roundtrip() { let input = [1u8; 64]; let mut output = [0u8; 64]; - asm!( + std::arch::asm!( "vmovups {zmm}, [{input}]", "vmovups [{output}], {zmm}", input = in(reg) input.as_ptr(), diff --git a/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch b/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch index 029e493227cd1..a2fcd97349e2b 100644 --- a/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch +++ b/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch @@ -154,6 +154,21 @@ index 510f4c9..175cbce 100644 -impl_trait! { f16 { bits: u16, mask: i16 }, f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } +impl_trait! { f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } +diff --git a/crates/core_simd/src/simd/prelude.rs b/crates/core_simd/src/simd/prelude.rs +index 51b8def..6e93f16 100644 +--- a/crates/core_simd/src/simd/prelude.rs ++++ b/crates/core_simd/src/simd/prelude.rs +@@ -14,10 +14,6 @@ pub use super::{ + simd_swizzle, + }; + +-#[rustfmt::skip] +-#[doc(no_inline)] +-pub use super::{f16x1, f16x2, f16x4, f16x8, f16x16, f16x32, f16x64}; +- + #[rustfmt::skip] + #[doc(no_inline)] + pub use super::{f32x1, f32x2, f32x4, f32x8, f32x16, f32x32, f32x64}; diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index fbef69f..c8e0b8c 100644 --- a/crates/core_simd/src/vector.rs diff --git a/patches/0027-stdlib-128bit-atomic-operations.patch b/patches/0027-stdlib-128bit-atomic-operations.patch index b7276e43153bc..717495cbcdf33 100644 --- a/patches/0027-stdlib-128bit-atomic-operations.patch +++ b/patches/0027-stdlib-128bit-atomic-operations.patch @@ -34,17 +34,17 @@ index a60f0799c0e..af056fbf41f 100644 #[cfg(target_has_atomic_load_store = "8")] #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs -index bf2b6d59f88..d5ccce03bbf 100644 +index 8a9a0b5..92ed9a6 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs -@@ -3585,42 +3585,6 @@ pub const fn as_ptr(&self) -> *mut $int_type { +@@ -3762,42 +3757,6 @@ atomic_int! { 8, u64 AtomicU64 } -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { - cfg(target_has_atomic = "128"), -- cfg(target_has_atomic_equal_alignment = "128"), +- cfg(target_has_atomic_primitive_alignment = "128"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), @@ -62,7 +62,7 @@ index bf2b6d59f88..d5ccce03bbf 100644 -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { - cfg(target_has_atomic = "128"), -- cfg(target_has_atomic_equal_alignment = "128"), +- cfg(target_has_atomic_primitive_alignment = "128"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), diff --git a/patches/0029-sysroot_tests-disable-f16-math.patch b/patches/0029-sysroot_tests-disable-f16-math.patch index 2aeb4a8a1874c..8d21359aa0043 100644 --- a/patches/0029-sysroot_tests-disable-f16-math.patch +++ b/patches/0029-sysroot_tests-disable-f16-math.patch @@ -7,11 +7,389 @@ Subject: [PATCH] Disable f16 math tests for cranelift coretests/tests/num/floats.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) -diff --git a/coretests/tests/floats/mod.rs b/coretests/tests/floats/mod.rs -index c61961f8584..d7b4fa20322 100644 +diff --git a/coretests/tests/num/floats.rs b/coretests/tests/num/floats.rs +index 1d7956b..01e4caa 100644 --- a/coretests/tests/num/floats.rs +++ b/coretests/tests/num/floats.rs -@@ -1534,7 +1534,7 @@ fn s_nan() -> Float { +@@ -444,7 +444,7 @@ pub(crate) use float_test; + float_test! { + name: num, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -463,7 +463,7 @@ float_test! { + name: num_rem, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -476,7 +476,7 @@ float_test! { + float_test! { + name: nan, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -496,7 +496,7 @@ float_test! { + float_test! { + name: infinity, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -514,7 +514,7 @@ float_test! { + float_test! { + name: neg_infinity, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -532,7 +532,7 @@ float_test! { + float_test! { + name: zero, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -550,7 +550,7 @@ float_test! { + float_test! { + name: neg_zero, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -570,7 +570,7 @@ float_test! { + float_test! { + name: one, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -588,7 +588,7 @@ float_test! { + float_test! { + name: is_nan, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -609,7 +609,7 @@ float_test! { + float_test! { + name: is_infinite, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -630,7 +630,7 @@ float_test! { + float_test! { + name: is_finite, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -651,7 +651,7 @@ float_test! { + float_test! { + name: is_normal, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -673,7 +673,7 @@ float_test! { + float_test! { + name: classify, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + }, + test { + let nan: Float = Float::NAN; +@@ -695,7 +695,7 @@ float_test! { + name: min, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -737,7 +737,7 @@ float_test! { + name: max, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -780,7 +780,7 @@ float_test! { + name: minimum, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -812,7 +812,7 @@ float_test! { + name: maximum, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -845,7 +845,7 @@ float_test! { + name: midpoint, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -898,7 +898,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Needs powi +- f16: #[cfg(target_has_reliable_f16_math)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128_math)], + }, + test { +@@ -929,7 +929,7 @@ float_test! { + name: abs, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -948,7 +948,7 @@ float_test! { + name: copysign, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -964,7 +964,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -982,7 +982,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -998,7 +998,7 @@ float_test! { + name: floor, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1028,7 +1028,7 @@ float_test! { + name: ceil, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1058,7 +1058,7 @@ float_test! { + name: round, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1089,7 +1089,7 @@ float_test! { + name: round_ties_even, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1120,7 +1120,7 @@ float_test! { + name: trunc, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1150,7 +1150,7 @@ float_test! { + name: fract, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1182,7 +1182,7 @@ float_test! { + name: signum, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1200,7 +1200,7 @@ float_test! { + float_test! { + name: is_sign_positive, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1219,7 +1219,7 @@ float_test! { + float_test! { + name: is_sign_negative, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1238,7 +1238,7 @@ float_test! { + float_test! { + name: next_up, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1269,7 +1269,7 @@ float_test! { + float_test! { + name: next_down, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1303,7 +1303,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1321,7 +1321,7 @@ float_test! { + name: clamp_min_greater_than_max, + attrs: { + const: #[cfg(false)], +- f16: #[should_panic, cfg(target_has_reliable_f16)], ++ f16: #[should_panic, cfg(false)], + f32: #[should_panic], + f64: #[should_panic], + f128: #[should_panic, cfg(target_has_reliable_f128)], +@@ -1335,7 +1335,7 @@ float_test! { + name: clamp_min_is_nan, + attrs: { + const: #[cfg(false)], +- f16: #[should_panic, cfg(target_has_reliable_f16)], ++ f16: #[should_panic, cfg(false)], + f32: #[should_panic], + f64: #[should_panic], + f128: #[should_panic, cfg(target_has_reliable_f128)], +@@ -1349,7 +1349,7 @@ float_test! { + name: clamp_max_is_nan, + attrs: { + const: #[cfg(false)], +- f16: #[should_panic, cfg(target_has_reliable_f16)], ++ f16: #[should_panic, cfg(false)], + f32: #[should_panic], + f64: #[should_panic], + f128: #[should_panic, cfg(target_has_reliable_f128)], +@@ -1363,7 +1363,7 @@ float_test! { + name: total_cmp, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1469,7 +1469,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1526,7 +1526,7 @@ float_test! { + name: recip, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1549,7 +1549,7 @@ float_test! { + name: powi, + attrs: { + const: #[cfg(false)], +- f16: #[cfg(target_has_reliable_f16_math)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128_math)], + }, + test { +@@ -1570,7 +1570,7 @@ float_test! { name: powf, attrs: { const: #[cfg(false)], @@ -20,7 +398,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1557,7 +1557,7 @@ fn s_nan() -> Float { +@@ -1593,7 +1593,7 @@ float_test! { name: exp, attrs: { const: #[cfg(false)], @@ -29,7 +407,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1578,7 +1578,7 @@ fn s_nan() -> Float { +@@ -1614,7 +1614,7 @@ float_test! { name: exp2, attrs: { const: #[cfg(false)], @@ -38,7 +416,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1598,7 +1598,7 @@ fn s_nan() -> Float { +@@ -1634,7 +1634,7 @@ float_test! { name: ln, attrs: { const: #[cfg(false)], @@ -47,7 +425,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1620,7 +1620,7 @@ fn s_nan() -> Float { +@@ -1656,7 +1656,7 @@ float_test! { name: log, attrs: { const: #[cfg(false)], @@ -56,7 +434,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1645,7 +1645,7 @@ fn s_nan() -> Float { +@@ -1681,7 +1681,7 @@ float_test! { name: log2, attrs: { const: #[cfg(false)], @@ -65,7 +443,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1668,7 +1668,7 @@ fn s_nan() -> Float { +@@ -1704,7 +1704,7 @@ float_test! { name: log10, attrs: { const: #[cfg(false)], @@ -74,7 +452,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1692,7 +1692,7 @@ fn s_nan() -> Float { +@@ -1728,7 +1728,7 @@ float_test! { name: asinh, attrs: { const: #[cfg(false)], @@ -83,7 +461,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1725,7 +1725,7 @@ fn s_nan() -> Float { +@@ -1764,7 +1764,7 @@ float_test! { name: acosh, attrs: { const: #[cfg(false)], @@ -92,7 +470,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1753,7 +1753,7 @@ fn s_nan() -> Float { +@@ -1795,7 +1795,7 @@ float_test! { name: atanh, attrs: { const: #[cfg(false)], @@ -101,7 +479,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1779,7 +1779,7 @@ fn s_nan() -> Float { +@@ -1821,7 +1821,7 @@ float_test! { name: gamma, attrs: { const: #[cfg(false)], @@ -110,7 +488,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1814,7 +1814,7 @@ fn s_nan() -> Float { +@@ -1856,7 +1856,7 @@ float_test! { name: ln_gamma, attrs: { const: #[cfg(false)], @@ -119,7 +497,79 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -2027,7 +2027,7 @@ fn s_nan() -> Float { +@@ -1874,7 +1874,7 @@ float_test! { + float_test! { + name: to_degrees, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1895,7 +1895,7 @@ float_test! { + float_test! { + name: to_radians, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1916,7 +1916,7 @@ float_test! { + float_test! { + name: to_algebraic, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1940,7 +1940,7 @@ float_test! { + float_test! { + name: to_bits_conv, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1967,7 +1967,7 @@ float_test! { + float_test! { + name: mul_add, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + // FIXME(#140515): mingw has an incorrect fma https://sourceforge.net/p/mingw-w64/bugs/848/ + f32: #[cfg_attr(all(target_os = "windows", target_env = "gnu", not(target_abi = "llvm")), ignore)], + f64: #[cfg_attr(all(target_os = "windows", target_env = "gnu", not(target_abi = "llvm")), ignore)], +@@ -1992,7 +1992,7 @@ float_test! { + float_test! { + name: from, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -2049,7 +2049,7 @@ float_test! { + float_test! { + name: max_exact_integer_constant, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -2091,7 +2091,7 @@ float_test! { + float_test! { + name: min_exact_integer_constant, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -2156,7 +2156,7 @@ float_test! { attrs: { // FIXME(f16_f128): add math tests when available const: #[cfg(false)], diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 486078185db84..faadf08929557 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-04-28" +channel = "nightly-2026-06-06" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index 7ce54a45e2ce6..ab31c43fb1b12 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -54,7 +54,7 @@ index 2e16f2cf27..3ac3df99a8 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -1147,6 +1147,8 @@ class RustBuild(object): - args += ["-Zwarnings"] + if deny_warnings: env["CARGO_BUILD_WARNINGS"] = "deny" + env["RUSTFLAGS"] += " -Zbinary-dep-depinfo" diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 4a663c48c0aff..683adeb49edd1 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -88,6 +88,8 @@ rm -r tests/run-make/reproducible-build-2 rm -r tests/run-make/no-builtins-lto rm -r tests/run-make/reachable-extern-fn-available-lto rm -r tests/run-make/no-builtins-linker-plugin-lto +rm -r tests/run-make/fat-then-thin-lto +rm -r tests/run-make/cross-lang-lto-upstream-rlibs # coverage instrumentation rm tests/ui/consts/precise-drop-with-coverage.rs @@ -146,6 +148,7 @@ rm tests/ui/consts/const-mut-refs-crate.rs # same rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift rm -r tests/run-make/short-ice # ICE backtrace begin/end marker mismatch rm -r tests/run-make/naked-dead-code-elimination # function not eliminated +rm tests/ui/codegen/huge-stacks.rs # Cranelift doesn't allow stack frames to exceed 4GB # doesn't work due to the way the rustc test suite is invoked. # should work when using ./x.py test the way it is intended diff --git a/src/abi/mod.rs b/src/abi/mod.rs index a8b179f169bf6..2ace6afd13e88 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -31,6 +31,11 @@ use crate::base::codegen_unwind_terminate; use crate::debuginfo::EXCEPTION_HANDLER_CLEANUP; use crate::prelude::*; +struct ArgValue<'tcx> { + value: CValue<'tcx>, + is_underaligned_pointee: bool, +} + fn clif_sig_from_fn_abi<'tcx>( tcx: TyCtxt<'tcx>, default_call_conv: CallConv, @@ -245,8 +250,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ // None means pass_mode == NoPass enum ArgKind<'tcx> { - Normal(Option>), - Spread(Vec>>), + Normal(Option>), + Spread(Vec>>), } // FIXME implement variadics in cranelift @@ -299,8 +304,12 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. let arg_abi = arg_abis_iter.next().unwrap(); - fx.caller_location = - Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap()); + let param = cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap(); + assert!( + !param.is_underaligned_pointee, + "caller location argument should not be underaligned", + ); + fx.caller_location = Some(param.value); } assert_eq!(arg_abis_iter.next(), None, "ArgAbi left behind for {:?}", fx.fn_abi); @@ -311,23 +320,24 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ for (local, arg_kind, ty) in func_params { // While this is normally an optimization to prevent an unnecessary copy when an argument is // not mutated by the current function, this is necessary to support unsized arguments. - if let ArgKind::Normal(Some(val)) = arg_kind { - if let Some((addr, meta)) = val.try_to_ptr() { - // Ownership of the value at the backing storage for an argument is passed to the - // callee per the ABI, so it is fine to borrow the backing storage of this argument - // to prevent a copy. - - let place = if let Some(meta) = meta { - CPlace::for_ptr_with_extra(addr, meta, val.layout()) - } else { - CPlace::for_ptr(addr, val.layout()) - }; + if let ArgKind::Normal(Some(ArgValue { value: val, is_underaligned_pointee: false })) = + arg_kind + && let Some((addr, meta)) = val.try_to_ptr() + { + // Ownership of the value at the backing storage for an argument is passed to the + // callee per the ABI, so it is fine to borrow the backing storage of this argument + // to prevent a copy. + + let place = if let Some(meta) = meta { + CPlace::for_ptr_with_extra(addr, meta, val.layout()) + } else { + CPlace::for_ptr(addr, val.layout()) + }; - self::comments::add_local_place_comments(fx, place, local); + self::comments::add_local_place_comments(fx, place, local); - assert_eq!(fx.local_map.push(place), local); - continue; - } + assert_eq!(fx.local_map.push(place), local); + continue; } let layout = fx.layout_of(ty); @@ -338,13 +348,22 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ match arg_kind { ArgKind::Normal(param) => { if let Some(param) = param { - place.write_cvalue(fx, param); + if param.is_underaligned_pointee { + place.write_cvalue_transmute(fx, param.value); + } else { + place.write_cvalue(fx, param.value); + } } } ArgKind::Spread(params) => { for (i, param) in params.into_iter().enumerate() { if let Some(param) = param { - place.place_field(fx, FieldIdx::new(i)).write_cvalue(fx, param); + let field_place = place.place_field(fx, FieldIdx::new(i)); + if param.is_underaligned_pointee { + field_place.write_cvalue_transmute(fx, param.value); + } else { + field_place.write_cvalue(fx, param.value); + } } } } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index edbee60471830..612f89e6a4217 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -7,6 +7,7 @@ use rustc_target::callconv::{ }; use smallvec::{SmallVec, smallvec}; +use super::ArgValue; use crate::prelude::*; use crate::value_and_place::assert_assignable; @@ -285,7 +286,7 @@ pub(super) fn cvalue_for_param<'tcx>( local_field: Option, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, block_params_iter: &mut impl Iterator, -) -> Option> { +) -> Option> { let block_params = arg_abi .get_abi_param(fx.tcx) .into_iter() @@ -306,30 +307,42 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, ); - match arg_abi.mode { - PassMode::Ignore => None, + let value = match arg_abi.mode { + PassMode::Ignore => return None, PassMode::Direct(_) => { assert_eq!(block_params.len(), 1, "{:?}", block_params); - Some(CValue::by_val(block_params[0], arg_abi.layout)) + CValue::by_val(block_params[0], arg_abi.layout) } PassMode::Pair(_, _) => { assert_eq!(block_params.len(), 2, "{:?}", block_params); - Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) + CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout) } PassMode::Cast { ref cast, .. } => { - Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) + from_casted_value(fx, &block_params, arg_abi.layout, cast) } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { assert_eq!(block_params.len(), 1, "{:?}", block_params); - Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) + if let Some(pointee_align) = attrs.pointee_align + && pointee_align < arg_abi.layout.align.abi + && arg_abi.layout.is_sized() + && arg_abi.layout.size != Size::ZERO + { + // Underaligned pointer: treat as `[u8; size]` and transmute-copy into the real type. + let bytes_ty = Ty::new_array(fx.tcx, fx.tcx.types.u8, arg_abi.layout.size.bytes()); + let bytes_layout = fx.layout_of(bytes_ty); + return Some(ArgValue { + value: CValue::by_ref(Pointer::new(block_params[0]), bytes_layout), + is_underaligned_pointee: true, + }); + } else { + CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout) + } } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { assert_eq!(block_params.len(), 2, "{:?}", block_params); - Some(CValue::by_ref_unsized( - Pointer::new(block_params[0]), - block_params[1], - arg_abi.layout, - )) + CValue::by_ref_unsized(Pointer::new(block_params[0]), block_params[1], arg_abi.layout) } - } + }; + + Some(ArgValue { value, is_underaligned_pointee: false }) } diff --git a/src/allocator.rs b/src/allocator.rs index 4a9b0c0952ff3..3c18748ee24b4 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -5,20 +5,11 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use rustc_ast::expand::allocator::{ AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, default_fn_name, global_fn_name, }; -use rustc_codegen_ssa::base::{allocator_kind_for_codegen, allocator_shim_contents}; use rustc_symbol_mangling::mangle_internal_symbol; use crate::prelude::*; -/// Returns whether an allocator shim was created -pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module) -> bool { - let Some(kind) = allocator_kind_for_codegen(tcx) else { return false }; - let methods = allocator_shim_contents(tcx, kind); - codegen_inner(tcx, module, &methods); - true -} - -fn codegen_inner(tcx: TyCtxt<'_>, module: &mut dyn Module, methods: &[AllocatorMethod]) { +pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module, methods: &[AllocatorMethod]) { let usize_ty = module.target_config().pointer_type(); for method in methods { diff --git a/src/analyze.rs b/src/analyze.rs index 72380f50385a1..c4a31cabcf385 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -23,14 +23,10 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec { for bb in fx.mir.basic_blocks.iter() { for stmt in bb.statements.iter() { - match &stmt.kind { - Assign(place_and_rval) => match &place_and_rval.1 { - Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) => { - flag_map[place.local] = SsaKind::NotSsa; - } - _ => {} - }, - _ => {} + if let Assign(place_and_rval) = &stmt.kind + && let Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) = &place_and_rval.1 + { + flag_map[place.local] = SsaKind::NotSsa; } } } diff --git a/src/base.rs b/src/base.rs index 2edbdb560f52b..467eceea221c8 100644 --- a/src/base.rs +++ b/src/base.rs @@ -147,7 +147,7 @@ pub(crate) fn codegen_fn<'tcx>( } pub(crate) fn compile_fn( - profiler: &SelfProfilerRef, + prof: &SelfProfilerRef, output_filenames: &OutputFilenames, should_write_ir: bool, cached_context: &mut Context, @@ -156,8 +156,7 @@ pub(crate) fn compile_fn( global_asm: &mut String, codegened_func: CodegenedFunction, ) { - let _timer = - profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); + let _timer = prof.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); let clif_comments = codegened_func.clif_comments; global_asm.push_str(&codegened_func.inline_asm); @@ -196,7 +195,7 @@ pub(crate) fn compile_fn( }; // Define function - profiler.generic_activity("define function").run(|| { + prof.generic_activity("define function").run(|| { context.want_disasm = should_write_ir; match module.define_function(codegened_func.func_id, context) { Ok(()) => {} @@ -248,7 +247,7 @@ pub(crate) fn compile_fn( } // Define debuginfo for function - profiler.generic_activity("generate debug info").run(|| { + prof.generic_activity("generate debug info").run(|| { if let Some(debug_context) = debug_context { codegened_func.func_debug_cx.unwrap().finalize( debug_context, @@ -1052,7 +1051,7 @@ pub(crate) fn codegen_operand<'tcx>( Operand::RuntimeChecks(checks) => { let val = checks.value(fx.tcx.sess); let layout = fx.layout_of(fx.tcx.types.bool); - return CValue::const_val(fx, layout, val.into()); + CValue::const_val(fx, layout, val.into()) } } } diff --git a/src/concurrency_limiter.rs b/src/concurrency_limiter.rs deleted file mode 100644 index b5a81fc11d57b..0000000000000 --- a/src/concurrency_limiter.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::sync::{Arc, Condvar, Mutex}; - -use rustc_data_structures::jobserver::{self, HelperThread}; -use rustc_errors::DiagCtxtHandle; - -// FIXME don't panic when a worker thread panics - -pub(super) struct ConcurrencyLimiter { - helper_thread: Option>, - state: Arc>, - available_token_condvar: Arc, - finished: bool, -} - -impl ConcurrencyLimiter { - pub(super) fn new(pending_jobs: usize) -> Self { - let state = Arc::new(Mutex::new(state::ConcurrencyLimiterState::new(pending_jobs))); - let available_token_condvar = Arc::new(Condvar::new()); - - let state_helper = state.clone(); - let available_token_condvar_helper = available_token_condvar.clone(); - let helper_thread = jobserver::client() - .clone() - .into_helper_thread(move |token| { - let mut state = state_helper.lock().unwrap(); - match token { - Ok(token) => { - state.add_new_token(token); - available_token_condvar_helper.notify_one(); - } - Err(err) => { - state.poison(format!("failed to acquire jobserver token: {}", err)); - // Notify all threads waiting for a token to give them a chance to - // gracefully exit. - available_token_condvar_helper.notify_all(); - } - } - }) - .unwrap(); - ConcurrencyLimiter { - helper_thread: Some(Mutex::new(helper_thread)), - state, - available_token_condvar, - finished: false, - } - } - - pub(super) fn acquire(&self, dcx: DiagCtxtHandle<'_>) -> ConcurrencyLimiterToken { - let mut state = self.state.lock().unwrap(); - loop { - state.assert_invariants(); - - match state.try_start_job() { - Ok(true) => { - return ConcurrencyLimiterToken { - state: self.state.clone(), - available_token_condvar: self.available_token_condvar.clone(), - }; - } - Ok(false) => {} - Err(err) => { - // An error happened when acquiring the token. Raise it as fatal error. - // Make sure to drop the mutex guard first to prevent poisoning the mutex. - drop(state); - if let Some(err) = err { - dcx.fatal(err); - } else { - // The error was already emitted, but compilation continued. Raise a silent - // fatal error. - rustc_errors::FatalError.raise(); - } - } - } - - self.helper_thread.as_ref().unwrap().lock().unwrap().request_token(); - state = self.available_token_condvar.wait(state).unwrap(); - } - } - - pub(crate) fn finished(mut self) { - self.helper_thread.take(); - - // Assert that all jobs have finished - let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap(); - state.assert_done(); - - self.finished = true; - } -} - -impl Drop for ConcurrencyLimiter { - fn drop(&mut self) { - if !self.finished && !std::thread::panicking() { - panic!("Forgot to call finished() on ConcurrencyLimiter"); - } - } -} - -#[derive(Debug)] -pub(super) struct ConcurrencyLimiterToken { - state: Arc>, - available_token_condvar: Arc, -} - -impl Drop for ConcurrencyLimiterToken { - fn drop(&mut self) { - let mut state = self.state.lock().unwrap(); - state.job_finished(); - self.available_token_condvar.notify_one(); - } -} - -mod state { - use rustc_data_structures::jobserver::Acquired; - - #[derive(Debug)] - pub(super) struct ConcurrencyLimiterState { - pending_jobs: usize, - active_jobs: usize, - - poisoned: bool, - stored_error: Option, - - // None is used to represent the implicit token, Some to represent explicit tokens - tokens: Vec>, - } - - impl ConcurrencyLimiterState { - pub(super) fn new(pending_jobs: usize) -> Self { - ConcurrencyLimiterState { - pending_jobs, - active_jobs: 0, - poisoned: false, - stored_error: None, - tokens: vec![None], - } - } - - pub(super) fn assert_invariants(&self) { - // There must be no excess active jobs - assert!(self.active_jobs <= self.pending_jobs); - - // There may not be more active jobs than there are tokens - assert!(self.active_jobs <= self.tokens.len()); - } - - pub(super) fn assert_done(&self) { - assert_eq!(self.pending_jobs, 0); - assert_eq!(self.active_jobs, 0); - } - - pub(super) fn add_new_token(&mut self, token: Acquired) { - self.tokens.push(Some(token)); - self.drop_excess_capacity(); - } - - pub(super) fn try_start_job(&mut self) -> Result> { - if self.poisoned { - return Err(self.stored_error.take()); - } - - if self.active_jobs < self.tokens.len() { - // Using existing token - self.job_started(); - return Ok(true); - } - - Ok(false) - } - - pub(super) fn job_started(&mut self) { - self.assert_invariants(); - self.active_jobs += 1; - self.drop_excess_capacity(); - self.assert_invariants(); - } - - pub(super) fn job_finished(&mut self) { - self.assert_invariants(); - self.pending_jobs -= 1; - self.active_jobs -= 1; - self.assert_invariants(); - self.drop_excess_capacity(); - self.assert_invariants(); - } - - pub(super) fn poison(&mut self, error: String) { - self.poisoned = true; - self.stored_error = Some(error); - } - - fn drop_excess_capacity(&mut self) { - self.assert_invariants(); - - // Drop all tokens that can never be used anymore - self.tokens.truncate(std::cmp::max(self.pending_jobs, 1)); - - // Keep some excess tokens to satisfy requests faster - const MAX_EXTRA_CAPACITY: usize = 2; - self.tokens.truncate(std::cmp::max(self.active_jobs + MAX_EXTRA_CAPACITY, 1)); - - self.assert_invariants(); - } - } -} diff --git a/src/constant.rs b/src/constant.rs index f85d21db11fb2..c986666f9c46e 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -455,7 +455,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant } else { ("", section_name.as_str()) }; - data.set_segment_section(segment_name, section_name); + // FIXME pass correct section flags on Mach-O + data.set_segment_section(segment_name, section_name, 0); } let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index 1ce424332db20..4b0260a8abc74 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -204,7 +204,7 @@ impl UnwindContext { let mut data = DataDescription::new(); data.define(gcc_except_table.writer.into_vec().into_boxed_slice()); - data.set_segment_section("", ".gcc_except_table"); + data.set_segment_section("", ".gcc_except_table", 0); for reloc in &gcc_except_table.relocs { match reloc.name { diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 323fec06bcc58..fcaf80d968a49 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -1,184 +1,103 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. -use std::env; +use std::convert::Infallible; use std::fs::File; use std::io::BufWriter; use std::path::PathBuf; use std::sync::Arc; -use std::thread::JoinHandle; +use std::time::Instant; use cranelift_object::{ObjectBuilder, ObjectModule}; -use rustc_codegen_ssa::assert_module_sources::CguReuse; -use rustc_codegen_ssa::back::write::produce_final_output_artifacts; -use rustc_codegen_ssa::base::determine_cgu_reuse; -use rustc_codegen_ssa::{CompiledModule, CompiledModules, ModuleKind}; +use rustc_ast::expand::allocator::AllocatorMethod; +use rustc_codegen_ssa::back::lto::ThinModule; +use rustc_codegen_ssa::back::write::{ + CodegenContext, FatLtoInput, ModuleConfig, SharedEmitter, TargetMachineFactoryFn, ThinLtoInput, +}; +use rustc_codegen_ssa::traits::{ExtraBackendMethods, WriteBackendMethods}; +use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_data_structures::stable_hash::{StableHash, StableHashCtxt, StableHasher}; -use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; +use rustc_errors::DiagCtxt; use rustc_hir::attrs::Linkage as RLinkage; -use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mono::{CodegenUnit, MonoItem, MonoItemData, Visibility}; +use rustc_middle::mono::{MonoItem, MonoItemData, Visibility}; use rustc_session::Session; -use rustc_session::config::{OutputFilenames, OutputType}; +use rustc_session::config::{OptLevel, OutputFilenames, OutputType}; +use rustc_span::Symbol; use crate::base::CodegenedFunction; -use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken}; use crate::debuginfo::TypeDebugContext; use crate::global_asm::{GlobalAsmConfig, GlobalAsmContext}; use crate::prelude::*; use crate::unwind_module::UnwindModule; -fn disable_incr_cache() -> bool { - env::var("CG_CLIF_DISABLE_INCR_CACHE").as_deref() == Ok("1") -} - -struct ModuleCodegenResult { - module: CompiledModule, - existing_work_product: Option<(WorkProductId, WorkProduct)>, -} - -enum OngoingModuleCodegen { - Sync(Result), - Async(JoinHandle>), -} - -impl StableHash for OngoingModuleCodegen { - fn stable_hash(&self, _: &mut Hcx, _: &mut StableHasher) { - // do nothing - } -} - -pub(crate) struct OngoingCodegen { - modules: Vec, - allocator_module: Option, - concurrency_limiter: ConcurrencyLimiter, -} - -impl OngoingCodegen { - pub(crate) fn join( - self, - sess: &Session, - outputs: &OutputFilenames, - ) -> (CompiledModules, FxIndexMap) { - let mut work_products = FxIndexMap::default(); - let mut modules = vec![]; - let disable_incr_cache = disable_incr_cache(); - - for module_codegen in self.modules { - let module_codegen_result = match module_codegen { - OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result, - OngoingModuleCodegen::Async(join_handle) => match join_handle.join() { - Ok(module_codegen_result) => module_codegen_result, - Err(panic) => std::panic::resume_unwind(panic), - }, - }; - - let module_codegen_result = match module_codegen_result { - Ok(module_codegen_result) => module_codegen_result, - Err(err) => sess.dcx().fatal(err), - }; - let ModuleCodegenResult { module, existing_work_product } = module_codegen_result; - - if let Some((work_product_id, work_product)) = existing_work_product { - work_products.insert(work_product_id, work_product); - } else { - let work_product = if disable_incr_cache { - None - } else if let Some(global_asm_object) = &module.global_asm_object { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - sess, - &module.name, - &[("o", module.object.as_ref().unwrap()), ("asm.o", global_asm_object)], - &[], - ) - } else { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - sess, - &module.name, - &[("o", module.object.as_ref().unwrap())], - &[], - ) - }; - if let Some((work_product_id, work_product)) = work_product { - work_products.insert(work_product_id, work_product); - } - } - - modules.push(module); - } - - self.concurrency_limiter.finished(); - - sess.dcx().abort_if_errors(); - - let compiled_modules = CompiledModules { modules, allocator_module: self.allocator_module }; - - produce_final_output_artifacts(sess, &compiled_modules, outputs); - - (compiled_modules, work_products) - } +pub(crate) struct AotModule { + producer: String, + global_asm_config: GlobalAsmConfig, + module: UnwindModule, + debug_context: Option, + codegened_functions: Vec, + global_asm: String, } -fn make_module(sess: &Session, name: String) -> UnwindModule { - let isa = crate::build_isa(sess, false); +fn make_module(tcx: TyCtxt<'_>, cgu_name: &str) -> AotModule { + let isa = crate::build_isa(tcx.sess, false); - let mut builder = - ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); + let mut builder = ObjectBuilder::new( + isa, + cgu_name.to_owned() + ".o", + cranelift_module::default_libcall_names(), + ) + .unwrap(); // Disable function sections by default on MSVC as it causes significant slowdowns with link.exe. // Maybe link.exe has exponential behavior when there are many sections with the same name? Also // explicitly disable it on MinGW as rustc already disables it by default on MinGW and as such // isn't tested. If rustc enables it in the future on MinGW, we can re-enable it too once it has // been on MinGW. - let default_function_sections = sess.target.function_sections && !sess.target.is_like_windows; + let default_function_sections = + tcx.sess.target.function_sections && !tcx.sess.target.is_like_windows; builder.per_function_section( - sess.opts.unstable_opts.function_sections.unwrap_or(default_function_sections), + tcx.sess.opts.unstable_opts.function_sections.unwrap_or(default_function_sections), ); - UnwindModule::new(ObjectModule::new(builder), true) -} + let module = UnwindModule::new(ObjectModule::new(builder), true); -fn emit_cgu( - output_filenames: &OutputFilenames, - prof: &SelfProfilerRef, - name: String, - module: UnwindModule, - debug: Option, - global_asm_object_file: Option, - producer: &str, -) -> Result { - let mut product = module.finish(); - - if let Some(mut debug) = debug { - debug.emit(&mut product); - } + let producer = crate::debuginfo::producer(tcx.sess); + let global_asm_config = GlobalAsmConfig::new(tcx.sess); + let debug_context = DebugContext::new(tcx, module.isa(), false, cgu_name); + let codegened_functions = vec![]; + let global_asm = String::new(); - let module = emit_module( - output_filenames, - prof, - product.object, - ModuleKind::Regular, - name.clone(), - global_asm_object_file, + AotModule { producer, - )?; - - Ok(ModuleCodegenResult { module, existing_work_product: None }) + global_asm_config, + module, + debug_context, + codegened_functions, + global_asm, + } } fn emit_module( output_filenames: &OutputFilenames, prof: &SelfProfilerRef, - mut object: cranelift_object::object::write::Object<'_>, + module: UnwindModule, + debug: Option, kind: ModuleKind, name: String, global_asm_object: Option, producer_str: &str, ) -> Result { - if object.format() == cranelift_object::object::BinaryFormat::Elf { - let comment_section = object.add_section( + let mut product = module.finish(); + + if let Some(mut debug) = debug { + debug.emit(&mut product); + } + + if product.object.format() == cranelift_object::object::BinaryFormat::Elf { + let comment_section = product.object.add_section( Vec::new(), b".comment".to_vec(), cranelift_object::object::SectionKind::OtherString, @@ -186,7 +105,7 @@ fn emit_module( let mut producer = vec![0]; producer.extend(producer_str.as_bytes()); producer.push(0); - object.set_section_data(comment_section, producer, 1); + product.object.set_section_data(comment_section, producer, 1); } let tmp_file = output_filenames.temp_path_for_cgu(OutputType::Object, &name); @@ -196,7 +115,7 @@ fn emit_module( }; let mut file = BufWriter::new(file); - if let Err(err) = object.write_stream(&mut file) { + if let Err(err) = product.object.write_stream(&mut file) { return Err(format!("error writing object file: {}", err)); } let file = match file.into_inner() { @@ -225,87 +144,22 @@ fn emit_module( }) } -fn reuse_workproduct_for_cgu( - tcx: TyCtxt<'_>, - cgu: &CodegenUnit<'_>, -) -> Result { - let work_product = cgu.previous_work_product(tcx); - let obj_out_regular = - tcx.output_filenames(()).temp_path_for_cgu(OutputType::Object, cgu.name().as_str()); - let source_file_regular = rustc_incremental::in_incr_comp_dir_sess( - tcx.sess, - work_product.saved_files.get("o").expect("no saved object file in work product"), - ); - - if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) { - return Err(format!( - "unable to copy {} to {}: {}", - source_file_regular.display(), - obj_out_regular.display(), - err - )); - } - - let obj_out_global_asm = - tcx.output_filenames(()).temp_path_ext_for_cgu("asm.o", cgu.name().as_str()); - let source_file_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") { - let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(tcx.sess, asm_o); - if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm) - { - return Err(format!( - "unable to copy {} to {}: {}", - source_file_global_asm.display(), - obj_out_global_asm.display(), - err - )); - } - Some(source_file_global_asm) - } else { - None - }; - - Ok(ModuleCodegenResult { - module: CompiledModule { - name: cgu.name().to_string(), - kind: ModuleKind::Regular, - object: Some(obj_out_regular), - global_asm_object: source_file_global_asm.as_ref().map(|_| obj_out_global_asm), - dwarf_object: None, - bytecode: None, - assembly: None, - llvm_ir: None, - links_from_incr_cache: if let Some(source_file_global_asm) = source_file_global_asm { - vec![source_file_regular, source_file_global_asm] - } else { - vec![source_file_regular] - }, - }, - existing_work_product: Some((cgu.work_product_id(), work_product)), - }) -} - -fn codegen_cgu_content( - tcx: TyCtxt<'_>, - module: &mut dyn Module, - cgu_name: rustc_span::Symbol, -) -> (Option, Vec, String) { +fn codegen_cgu(tcx: TyCtxt<'_>, cgu_name: Symbol) -> AotModule { let _timer = tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str()); let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let mut debug_context = DebugContext::new(tcx, module.isa(), false, cgu_name.as_str()); - let mut global_asm = String::new(); + let mut module = make_module(tcx, cgu_name.as_str()); let mut type_dbg = TypeDebugContext::default(); - super::predefine_mono_items(tcx, module, &mono_items); - let mut codegened_functions = vec![]; + super::predefine_mono_items(tcx, &mut module.module, &mono_items); for (mono_item, item_data) in mono_items { match mono_item { MonoItem::Fn(instance) => { let flags = tcx.codegen_instance_attrs(instance.def).flags; if flags.contains(CodegenFnAttrFlags::NAKED) { rustc_codegen_ssa::mir::naked_asm::codegen_naked_asm( - &mut GlobalAsmContext { tcx, global_asm: &mut global_asm }, + &mut GlobalAsmContext { tcx, global_asm: &mut module.global_asm }, instance, MonoItemData { linkage: RLinkage::External, @@ -321,186 +175,219 @@ fn codegen_cgu_content( } let codegened_function = crate::base::codegen_fn( tcx, - cgu_name, - debug_context.as_mut(), + cgu.name(), + module.debug_context.as_mut(), &mut type_dbg, Function::new(), - module, + &mut module.module, instance, ); - codegened_functions.push(codegened_function); + module.codegened_functions.push(codegened_function); } MonoItem::Static(def_id) => { - let data_id = crate::constant::codegen_static(tcx, module, def_id); - if let Some(debug_context) = debug_context.as_mut() { + let data_id = crate::constant::codegen_static(tcx, &mut module.module, def_id); + if let Some(debug_context) = module.debug_context.as_mut() { debug_context.define_static(tcx, &mut type_dbg, def_id, data_id); } } MonoItem::GlobalAsm(item_id) => { rustc_codegen_ssa::base::codegen_global_asm( - &mut GlobalAsmContext { tcx, global_asm: &mut global_asm }, + &mut GlobalAsmContext { tcx, global_asm: &mut module.global_asm }, item_id, ); } } } - crate::main_shim::maybe_create_entry_wrapper(tcx, module, false, cgu.is_primary()); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module.module, false, cgu.is_primary()); - (debug_context, codegened_functions, global_asm) + module } -fn module_codegen( - tcx: TyCtxt<'_>, - global_asm_config: Arc, - cgu_name: rustc_span::Symbol, - token: ConcurrencyLimiterToken, -) -> OngoingModuleCodegen { - let mut module = make_module(tcx.sess, cgu_name.as_str().to_string()); +fn compile_cgu( + prof: &SelfProfilerRef, + output_filenames: &OutputFilenames, + should_write_ir: bool, + mut aot_module: AotModule, + cgu_name: String, + kind: ModuleKind, +) -> Result { + prof.generic_activity_with_arg("compile functions", &*cgu_name).run(|| { + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( + prof.clone(), + ))); + + let mut cached_context = Context::new(); + for codegened_func in aot_module.codegened_functions { + crate::base::compile_fn( + &prof, + &output_filenames, + should_write_ir, + &mut cached_context, + &mut aot_module.module, + aot_module.debug_context.as_mut(), + &mut aot_module.global_asm, + codegened_func, + ); + } + }); - let (mut debug_context, codegened_functions, mut global_asm) = - codegen_cgu_content(tcx, &mut module, cgu_name); + let global_asm_object_file = + prof.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { + if aot_module.global_asm.is_empty() { + return Ok::<_, String>(None); + } - let cgu_name = cgu_name.as_str().to_owned(); + let global_asm_object_file = output_filenames.temp_path_ext_for_cgu("asm.o", &cgu_name); + crate::global_asm::compile_global_asm( + &aot_module.global_asm_config, + aot_module.global_asm, + &global_asm_object_file, + )?; + + Ok(Some(global_asm_object_file)) + })?; + + prof.generic_activity_with_arg("write object file", &*cgu_name).run(|| { + emit_module( + output_filenames, + prof, + aot_module.module, + aot_module.debug_context, + kind, + cgu_name.clone(), + global_asm_object_file, + &aot_module.producer, + ) + }) +} - let producer = crate::debuginfo::producer(tcx.sess); +#[derive(Copy, Clone)] +pub(crate) struct AotDriver; + +impl ExtraBackendMethods for AotDriver { + type Module = AotModule; + + fn codegen_allocator<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + module_name: &str, + methods: &[AllocatorMethod], + ) -> Self::Module { + let mut allocator_module = make_module(tcx, module_name); + crate::allocator::codegen(tcx, &mut allocator_module.module, methods); + allocator_module + } - let profiler = tcx.prof.clone(); - let output_filenames = tcx.output_filenames(()).clone(); - let should_write_ir = crate::pretty_clif::should_write_ir(tcx.sess); - - OngoingModuleCodegen::Async(std::thread::spawn(move || { - profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| { - cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( - profiler.clone(), - ))); - - let mut cached_context = Context::new(); - for codegened_func in codegened_functions { - crate::base::compile_fn( - &profiler, - &output_filenames, - should_write_ir, - &mut cached_context, - &mut module, - debug_context.as_mut(), - &mut global_asm, - codegened_func, - ); - } - }); - - let global_asm_object_file = - profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { - crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, global_asm) - })?; - - let codegen_result = - profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| { - emit_cgu( - &global_asm_config.output_filenames, - &profiler, - cgu_name, - module, - debug_context, - global_asm_object_file, - &producer, - ) - }); - std::mem::drop(token); - codegen_result - })) -} + fn compile_codegen_unit( + &self, + tcx: TyCtxt<'_>, + cgu_name: Symbol, + ) -> (ModuleCodegen, u64) { + let start_time = Instant::now(); + + let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); + let (module, _) = tcx.dep_graph.with_task( + dep_node, + tcx, + || { + let aot_module = codegen_cgu(tcx, cgu_name); + ModuleCodegen::new_regular(cgu_name.as_str().to_owned(), aot_module) + }, + Some(rustc_middle::dep_graph::hash_result), + ); -fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option { - let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string()); - let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module); - - if created_alloc_shim { - let product = allocator_module.finish(); - - match emit_module( - tcx.output_filenames(()), - &tcx.sess.prof, - product.object, - ModuleKind::Allocator, - "allocator_shim".to_owned(), - None, - &crate::debuginfo::producer(tcx.sess), - ) { - Ok(allocator_module) => Some(allocator_module), - Err(err) => tcx.dcx().fatal(err), - } - } else { - None + let time_to_codegen = start_time.elapsed(); + + // We assume that the cost to run LLVM on a CGU is proportional to + // the time we needed for codegenning it. + let cost = time_to_codegen.as_nanos() as u64; + + (module, cost) } } -pub(crate) fn run_aot(tcx: TyCtxt<'_>) -> Box { - let cgus = tcx.collect_and_partition_mono_items(()).codegen_units; +impl WriteBackendMethods for AotDriver { + type Module = AotModule; - if tcx.dep_graph.is_fully_enabled() { - for cgu in cgus { - tcx.ensure_ok().codegen_unit(cgu.name()); - } + type TargetMachine = (); + + type ModuleBuffer = Infallible; + + type ThinData = Infallible; + + fn target_machine_factory( + &self, + _sess: &Session, + _opt_level: OptLevel, + _target_features: &[String], + ) -> TargetMachineFactoryFn { + Arc::new(|_, _| ()) } - // Calculate the CGU reuse - let cgu_reuse = tcx.sess.time("find_cgu_reuse", || { - cgus.iter().map(|cgu| determine_cgu_reuse(tcx, cgu)).collect::>() - }); + fn optimize_and_codegen_fat_lto( + _sess: &Session, + _cgcx: &CodegenContext, + _shared_emitter: &SharedEmitter, + _tm_factory: TargetMachineFactoryFn, + _exported_symbols_for_lto: &[String], + _each_linked_rlib_for_lto: &[PathBuf], + _modules: Vec>, + ) -> CompiledModule { + unreachable!() + } - rustc_codegen_ssa::assert_module_sources::assert_module_sources(tcx, &|cgu_reuse_tracker| { - for (i, cgu) in cgus.iter().enumerate() { - let cgu_reuse = cgu_reuse[i]; - cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); - } - }); + fn run_thin_lto( + _cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _dcx: rustc_errors::DiagCtxtHandle<'_>, + _exported_symbols_for_lto: &[String], + _each_linked_rlib_for_lto: &[PathBuf], + _modules: Vec>, + ) -> (Vec>, Vec) { + unreachable!() + } - let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx)); + fn optimize( + _cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _shared_emitter: &SharedEmitter, + _module: &mut ModuleCodegen, + _config: &ModuleConfig, + ) { + } - let disable_incr_cache = disable_incr_cache(); - let (todo_cgus, done_cgus) = - cgus.iter().enumerate().partition::, _>(|&(i, _)| match cgu_reuse[i] { - _ if disable_incr_cache => true, - CguReuse::No => true, - CguReuse::PreLto | CguReuse::PostLto => false, - }); + fn optimize_and_codegen_thin( + _cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _shared_emitter: &SharedEmitter, + _tm_factory: TargetMachineFactoryFn, + _thin: ThinModule, + ) -> CompiledModule { + unreachable!() + } - let concurrency_limiter = IntoDynSyncSend(ConcurrencyLimiter::new(todo_cgus.len())); + fn codegen( + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &SharedEmitter, + module: ModuleCodegen, + config: &ModuleConfig, + ) -> CompiledModule { + compile_cgu( + prof, + &cgcx.output_filenames, + config.emit_ir, + module.module_llvm, + module.name, + module.kind, + ) + .unwrap_or_else(|err| { + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); + dcx.handle().fatal(err) + }) + } - let modules: Vec<_> = - tcx.sess.time("codegen mono items", || { - let modules: Vec<_> = par_map(todo_cgus, |(_, cgu)| { - let dep_node = cgu.codegen_dep_node(tcx); - let (module, _) = tcx.dep_graph.with_task( - dep_node, - tcx, - || { - module_codegen( - tcx, - global_asm_config.clone(), - cgu.name(), - concurrency_limiter.acquire(tcx.dcx()), - ) - }, - Some(rustc_middle::dep_graph::hash_result), - ); - IntoDynSyncSend(module) - }); - modules - .into_iter() - .map(|module| module.0) - .chain(done_cgus.into_iter().map(|(_, cgu)| { - OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu)) - })) - .collect() - }); - - let allocator_module = emit_allocator_module(tcx); - - Box::new(OngoingCodegen { - modules, - allocator_module, - concurrency_limiter: concurrency_limiter.0, - }) + fn serialize_module(_module: Self::Module, _is_thin: bool) -> Self::ModuleBuffer { + unreachable!() + } } diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 33b88d70d6f8d..7d6ece02e4a62 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -6,6 +6,7 @@ use std::os::raw::{c_char, c_int}; use cranelift_jit::{JITBuilder, JITModule}; use rustc_codegen_ssa::CrateInfo; +use rustc_codegen_ssa::base::{allocator_kind_for_codegen, allocator_shim_contents}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mono::MonoItem; use rustc_session::Session; @@ -28,7 +29,9 @@ fn create_jit_module( let cx = DebugContext::new(tcx, jit_module.isa(), false, "dummy_cgu_name"); - crate::allocator::codegen(tcx, &mut jit_module); + if let Some(kind) = allocator_kind_for_codegen(tcx) { + crate::allocator::codegen(tcx, &mut jit_module, &allocator_shim_contents(tcx, kind)); + } (jit_module, cx) } diff --git a/src/global_asm.rs b/src/global_asm.rs index ee7e6732e6a77..769c008c13f75 100644 --- a/src/global_asm.rs +++ b/src/global_asm.rs @@ -2,9 +2,8 @@ //! standalone executable. use std::io::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::Arc; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::traits::{AsmCodegenMethods, GlobalAsmOperandRef}; @@ -12,7 +11,7 @@ use rustc_middle::ty::TyCtxt; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers, }; -use rustc_session::config::OutputFilenames; +use rustc_session::Session; use rustc_target::asm::InlineAsmArch; use crate::prelude::*; @@ -163,32 +162,28 @@ fn codegen_global_asm_inner<'tcx>( pub(crate) struct GlobalAsmConfig { assembler: PathBuf, target: String, - pub(crate) output_filenames: Arc, } impl GlobalAsmConfig { - pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { + pub(crate) fn new(sess: &Session) -> Self { GlobalAsmConfig { - assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"), - target: match &tcx.sess.opts.target_triple { + assembler: crate::toolchain::get_toolchain_binary(sess, "as"), + target: match &sess.opts.target_triple { rustc_target::spec::TargetTuple::TargetTuple(triple) => triple.clone(), rustc_target::spec::TargetTuple::TargetJson { path_for_rustdoc, .. } => { path_for_rustdoc.to_str().unwrap().to_owned() } }, - output_filenames: tcx.output_filenames(()).clone(), } } } pub(crate) fn compile_global_asm( config: &GlobalAsmConfig, - cgu_name: &str, global_asm: String, -) -> Result, String> { - if global_asm.is_empty() { - return Ok(None); - } + global_asm_object_file: &Path, +) -> Result<(), String> { + assert!(!global_asm.is_empty()); // Remove all LLVM style comments let mut global_asm = global_asm @@ -198,8 +193,6 @@ pub(crate) fn compile_global_asm( .join("\n"); global_asm.push('\n'); - let global_asm_object_file = config.output_filenames.temp_path_ext_for_cgu("asm.o", cgu_name); - // Assemble `global_asm` if option_env!("CG_CLIF_FORCE_GNU_AS").is_some() { let mut child = Command::new(&config.assembler) @@ -266,5 +259,5 @@ pub(crate) fn compile_global_asm( } } - Ok(Some(global_asm_object_file)) + Ok(()) } diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 8100d565b3974..f9fc7002be87b 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -424,13 +424,10 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate stack slots for inout for (i, operand) in self.operands.iter().enumerate() { - match *operand { - CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => { - let slot = new_slot(reg.reg_class()); - slots_input[i] = Some(slot); - slots_output[i] = Some(slot); - } - _ => (), + if let CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } = *operand { + let slot = new_slot(reg.reg_class()); + slots_input[i] = Some(slot); + slots_output[i] = Some(slot); } } @@ -456,11 +453,8 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate stack slots for output for (i, operand) in self.operands.iter().enumerate() { - match *operand { - CInlineAsmOperand::Out { reg, place: Some(_), .. } => { - slots_output[i] = Some(new_slot(reg.reg_class())); - } - _ => (), + if let CInlineAsmOperand::Out { reg, place: Some(_), .. } = *operand { + slots_output[i] = Some(new_slot(reg.reg_class())); } } diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index 3cd7ebb88f447..9da87a5774e8f 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -494,6 +494,385 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( }); } */ + "llvm.aarch64.crc32b" + | "llvm.aarch64.crc32h" + | "llvm.aarch64.crc32w" + | "llvm.aarch64.crc32x" + | "llvm.aarch64.crc32cb" + | "llvm.aarch64.crc32ch" + | "llvm.aarch64.crc32cw" + | "llvm.aarch64.crc32cx" => { + // ARM ARM v8-A: CRC32{,C}{B,H,W,X}. + // Backs core::arch::aarch64::__crc32{,c}{b,h,w,d}. + intrinsic_args!(fx, args => (crc, v); intrinsic); + + let crc = crc.load_scalar(fx); + let v = v.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crc32b" => "crc32b w0, w0, w1", + "llvm.aarch64.crc32h" => "crc32h w0, w0, w1", + "llvm.aarch64.crc32w" => "crc32w w0, w0, w1", + "llvm.aarch64.crc32x" => "crc32x w0, w0, x1", + "llvm.aarch64.crc32cb" => "crc32cb w0, w0, w1", + "llvm.aarch64.crc32ch" => "crc32ch w0, w0, w1", + "llvm.aarch64.crc32cw" => "crc32cw w0, w0, w1", + "llvm.aarch64.crc32cx" => "crc32cx w0, w0, x1", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + _late: true, + in_value: crc, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: v, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.aese" | "llvm.aarch64.crypto.aesd" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.aese" => "aese v0.16b, v1.16b", + "llvm.aarch64.crypto.aesd" => "aesd v0.16b, v1.16b", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.aesmc" | "llvm.aarch64.crypto.aesimc" => { + intrinsic_args!(fx, args => (a); intrinsic); + + let a = a.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.aesmc" => "aesmc v0.16b, v0.16b", + "llvm.aarch64.crypto.aesimc" => "aesimc v0.16b, v0.16b", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha256h" | "llvm.aarch64.crypto.sha256h2" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.sha256h" => "sha256h q0, q1, v2.4s", + "llvm.aarch64.crypto.sha256h2" => "sha256h2 q0, q1, v2.4s", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + value: c, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha256su0" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha256su0 v0.4s, v1.4s".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha256su1" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha256su1 v0.4s, v1.4s, v2.4s".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + value: c, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.neon.pmull64" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + "fmov d0, x0 + fmov d1, x1 + pmull v0.1q, v0.1d, v1.1d" + .into(), + )], + &[ + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + late: true, + place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + value: a, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: b, + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + late: true, + place: None, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.neon.pmull.v8i16" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("pmull v0.8h, v0.8b, v1.8b".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.neon.sqdmulh.v2i32" + | "llvm.aarch64.neon.sqdmulh.v4i16" + | "llvm.aarch64.neon.sqdmulh.v4i32" + | "llvm.aarch64.neon.sqdmulh.v8i16" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/SQDMULH--vector---Signed-saturating-doubling-multiply-returning-high-half- + intrinsic_args!(fx, args => (a, b); intrinsic); + + // Simplify the "double and shift by esize" into "shift by esize - 1". + // https://github.com/qemu/qemu/blob/81cc5f39aa3042e9c0b2ea772b42a2c8b1488e76/target/arm/tcg/mve_helper.c#L1267-L1283 + let (result_ty, product_ty, shift, max) = match intrinsic { + "llvm.aarch64.neon.sqdmulh.v4i16" | "llvm.aarch64.neon.sqdmulh.v8i16" => { + (types::I16, types::I32, 15, i64::from(i16::MAX)) + } + "llvm.aarch64.neon.sqdmulh.v2i32" | "llvm.aarch64.neon.sqdmulh.v4i32" => { + (types::I32, types::I64, 31, i64::from(i32::MAX)) + } + _ => unreachable!(), + }; + + simd_pair_for_each_lane( + fx, + a, + b, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| { + let a_lane = fx.bcx.ins().sextend(product_ty, a_lane); + let b_lane = fx.bcx.ins().sextend(product_ty, b_lane); + let product = fx.bcx.ins().imul(a_lane, b_lane); + let product = fx.bcx.ins().sshr_imm(product, shift); + let max = fx.bcx.ins().iconst(product_ty, max); + let result = fx.bcx.ins().smin(product, max); + fx.bcx.ins().ireduce(result_ty, result) + }, + ); + } + + "llvm.aarch64.neon.saddlp.v1i64.v2i32" + | "llvm.aarch64.neon.saddlp.v2i32.v4i16" + | "llvm.aarch64.neon.saddlp.v2i64.v4i32" + | "llvm.aarch64.neon.saddlp.v4i16.v8i8" + | "llvm.aarch64.neon.saddlp.v4i32.v8i16" + | "llvm.aarch64.neon.saddlp.v8i16.v16i8" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/SADDLP--Signed-add-long-pairwise- + intrinsic_args!(fx, args => (a); intrinsic); + + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + let wide_ty = fx.clif_type(ret_lane_ty).unwrap(); + + for lane_idx in 0..ret_lane_count { + let base = lane_idx * 2; + let a_lane0 = a.value_lane(fx, base).load_scalar(fx); + let a_lane1 = a.value_lane(fx, base + 1).load_scalar(fx); + let a_lane0 = fx.bcx.ins().sextend(wide_ty, a_lane0); + let a_lane1 = fx.bcx.ins().sextend(wide_ty, a_lane1); + let sum = fx.bcx.ins().iadd(a_lane0, a_lane1); + let res_lane = CValue::by_val(sum, ret_lane_layout); + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } + } + + "llvm.aarch64.neon.uaddlp.v1i64.v2i32" + | "llvm.aarch64.neon.uaddlp.v2i32.v4i16" + | "llvm.aarch64.neon.uaddlp.v2i64.v4i32" + | "llvm.aarch64.neon.uaddlp.v4i16.v8i8" + | "llvm.aarch64.neon.uaddlp.v4i32.v8i16" + | "llvm.aarch64.neon.uaddlp.v8i16.v16i8" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/UADDLP--Unsigned-add-long-pairwise- + intrinsic_args!(fx, args => (a); intrinsic); + + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + let wide_ty = fx.clif_type(ret_lane_ty).unwrap(); + + for lane_idx in 0..ret_lane_count { + let base = lane_idx * 2; + let a_lane0 = a.value_lane(fx, base).load_scalar(fx); + let a_lane1 = a.value_lane(fx, base + 1).load_scalar(fx); + let a_lane0 = fx.bcx.ins().uextend(wide_ty, a_lane0); + let a_lane1 = fx.bcx.ins().uextend(wide_ty, a_lane1); + let sum = fx.bcx.ins().iadd(a_lane0, a_lane1); + let res_lane = CValue::by_val(sum, ret_lane_layout); + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } + } + _ => { fx.tcx.dcx().warn(format!( "unsupported AArch64 llvm intrinsic {}; replacing with trap", diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index b8e1886b2d3c1..5d7e457b8e1ba 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -371,7 +371,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } for i in 0..lane_count { - let ret_lane = ret.place_lane(fx, i.into()); + let ret_lane = ret.place_lane(fx, i); ret_lane.write_cvalue(fx, value); } } diff --git a/src/lib.rs b/src/lib.rs index 48858e20381dd..38cb986034859 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,9 +18,7 @@ extern crate rustc_codegen_ssa; extern crate rustc_const_eval; extern crate rustc_data_structures; extern crate rustc_errors; -extern crate rustc_fs_util; extern crate rustc_hir; -extern crate rustc_incremental; extern crate rustc_index; extern crate rustc_log; extern crate rustc_session; @@ -40,7 +38,7 @@ use std::sync::Arc; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; use rustc_codegen_ssa::traits::CodegenBackend; -use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig}; +use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig, back}; use rustc_log::tracing::info; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; @@ -60,7 +58,6 @@ mod codegen_f16_f128; mod codegen_i128; mod common; mod compiler_builtins; -mod concurrency_limiter; mod config; mod constant; mod debuginfo; @@ -133,7 +130,7 @@ impl CodegenBackend for CraneliftCodegenBackend { match sess.lto() { Lto::No | Lto::ThinLocal => {} Lto::Thin | Lto::Fat => { - sess.dcx().warn("LTO is not supported. You may get a linker error.") + sess.dcx().fatal("LTO is not supported by rustc_codegen_cranelift"); } } @@ -152,6 +149,10 @@ impl CodegenBackend for CraneliftCodegenBackend { } } + fn thin_lto_supported(&self) -> bool { + false + } + fn target_config(&self, sess: &Session) -> TargetConfig { // FIXME return the actually used target features. this is necessary for #[cfg(target_feature)] let target_features = match sess.target.arch { @@ -224,7 +225,7 @@ impl CodegenBackend for CraneliftCodegenBackend { #[cfg(not(feature = "jit"))] tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); } else { - driver::aot::run_aot(tcx) + Box::new(rustc_codegen_ssa::base::codegen_crate(driver::aot::AotDriver, tcx)) } } @@ -232,10 +233,13 @@ impl CodegenBackend for CraneliftCodegenBackend { &self, ongoing_codegen: Box, sess: &Session, - outputs: &OutputFilenames, - _crate_info: &CrateInfo, + _outputs: &OutputFilenames, + crate_info: &CrateInfo, ) -> (CompiledModules, FxIndexMap) { - ongoing_codegen.downcast::().unwrap().join(sess, outputs) + ongoing_codegen + .downcast::>() + .unwrap() + .join(sess, crate_info) } fn fallback_intrinsics(&self) -> Vec { @@ -254,7 +258,9 @@ fn enable_verifier(sess: &Session) -> bool { } fn target_triple(sess: &Session) -> target_lexicon::Triple { - match sess.target.llvm_target.parse() { + // Use versioned target triple to make `OperatingSystem::MacOSX(...)` + // contain a value, which we use when emitting `LC_BUILD_VERSION`. + match back::versioned_llvm_target(sess).parse() { Ok(triple) => triple, Err(err) => sess.dcx().fatal(format!("target not recognized: {}", err)), } diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 1d9dcdab4ced9..a2a2cac3faaa8 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -686,17 +686,14 @@ impl<'tcx> CPlace<'tcx> { ) -> CPlace<'tcx> { let layout = self.layout(); - match self.inner { - CPlaceInner::VarPair(local, var1, var2) => { - let layout = layout.field(&*fx, field.index()); + if let CPlaceInner::VarPair(local, var1, var2) = self.inner { + let layout = layout.field(&*fx, field.index()); - match field.as_u32() { - 0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout }, - 1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout }, - _ => unreachable!("field should be 0 or 1"), - } + match field.as_u32() { + 0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout }, + 1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout }, + _ => unreachable!("field should be 0 or 1"), } - _ => {} } let (base, extra) = match self.inner { From 7c540e53506e763f453d52047ebbb3cfe1324a3f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 3 Jun 2026 15:01:07 +0000 Subject: [PATCH 04/51] Use WorkProductMap instead of FxIndexMap This is an UnordMap internally. Iteration order for the work product map should not matter aside from the place it is serialized where sorting by WorkProductId is sufficient. --- src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 38cb986034859..af783f31a2136 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,7 @@ use cranelift_codegen::settings::{self, Configurable}; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig, back}; use rustc_log::tracing::info; -use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::dep_graph::WorkProductMap; use rustc_session::Session; use rustc_session::config::OutputFilenames; use rustc_span::{Symbol, sym}; @@ -88,7 +88,7 @@ mod prelude { }; pub(crate) use cranelift_module::{self, DataDescription, FuncId, Linkage, Module}; pub(crate) use rustc_abi::{BackendRepr, FIRST_VARIANT, FieldIdx, Scalar, Size, VariantIdx}; - pub(crate) use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; + pub(crate) use rustc_data_structures::fx::FxHashMap; pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE}; pub(crate) use rustc_index::Idx; pub(crate) use rustc_middle::mir::{self, *}; @@ -235,7 +235,7 @@ impl CodegenBackend for CraneliftCodegenBackend { sess: &Session, _outputs: &OutputFilenames, crate_info: &CrateInfo, - ) -> (CompiledModules, FxIndexMap) { + ) -> (CompiledModules, WorkProductMap) { ongoing_codegen .downcast::>() .unwrap() From 2c81a777737ca607f199d04fd7509c5df15f9147 Mon Sep 17 00:00:00 2001 From: Cathal Mullan Date: Wed, 10 Jun 2026 19:39:10 +0100 Subject: [PATCH 05/51] Implement remaining aarch64 SHA-3 LLVM intrinsics --- example/neon.rs | 235 +++++++++++++++++++++++++++++++++ src/intrinsics/llvm_aarch64.rs | 82 ++++++++++++ 2 files changed, 317 insertions(+) diff --git a/example/neon.rs b/example/neon.rs index 6b024de7bb560..ba63333daa339 100644 --- a/example/neon.rs +++ b/example/neon.rs @@ -470,6 +470,220 @@ unsafe fn test_vsha512su1q_u64() { assert_eq!(r, e); } +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_s8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v16i8 + let a = i8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let c = i8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); + let e = i8x16::from([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]); + let r: i8x16 = unsafe { transmute(veor3q_s8(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v8i16 + let a = i16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); + let b = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let c = i16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); + let e = i16x8::from([24, 25, 26, 27, 28, 29, 30, 31]); + let r: i16x8 = unsafe { transmute(veor3q_s16(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v4i32 + let a = i32x4::from([0, 1, 2, 3]); + let b = i32x4::from([4, 5, 6, 7]); + let c = i32x4::from([8, 9, 10, 11]); + let e = i32x4::from([12, 13, 14, 15]); + let r: i32x4 = unsafe { transmute(veor3q_s32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_s64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v2i64 + let a = i64x2::from([0, 1]); + let b = i64x2::from([2, 3]); + let c = i64x2::from([4, 5]); + let e = i64x2::from([6, 7]); + let r: i64x2 = unsafe { transmute(veor3q_s64(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v16i8 + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let c = u8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); + let e = u8x16::from([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]); + let r: u8x16 = unsafe { transmute(veor3q_u8(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_u16() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v8i16 + let a = u16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); + let b = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let c = u16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); + let e = u16x8::from([24, 25, 26, 27, 28, 29, 30, 31]); + let r: u16x8 = unsafe { transmute(veor3q_u16(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v4i32 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([12, 13, 14, 15]); + let r: u32x4 = unsafe { transmute(veor3q_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_veor3q_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v2i64 + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let c = u64x2::from([4, 5]); + let e = u64x2::from([6, 7]); + let r: u64x2 = unsafe { transmute(veor3q_u64(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_s8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v16i8 + let a = i8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let c = i8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); + let e = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let r: i8x16 = unsafe { transmute(vbcaxq_s8(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v8i16 + let a = i16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); + let b = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let c = i16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); + let e = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let r: i16x8 = unsafe { transmute(vbcaxq_s16(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v4i32 + let a = i32x4::from([0, 1, 2, 3]); + let b = i32x4::from([4, 5, 6, 7]); + let c = i32x4::from([8, 9, 10, 11]); + let e = i32x4::from([4, 5, 6, 7]); + let r: i32x4 = unsafe { transmute(vbcaxq_s32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_s64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v2i64 + let a = i64x2::from([0, 1]); + let b = i64x2::from([2, 3]); + let c = i64x2::from([4, 5]); + let e = i64x2::from([2, 3]); + let r: i64x2 = unsafe { transmute(vbcaxq_s64(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v16i8 + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let c = u8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]); + let e = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let r: u8x16 = unsafe { transmute(vbcaxq_u8(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_u16() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v8i16 + let a = u16x8::from([0, 1, 2, 3, 4, 5, 6, 7]); + let b = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let c = u16x8::from([16, 17, 18, 19, 20, 21, 22, 23]); + let e = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let r: u16x8 = unsafe { transmute(vbcaxq_u16(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v4i32 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([4, 5, 6, 7]); + let r: u32x4 = unsafe { transmute(vbcaxq_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vbcaxq_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v2i64 + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let c = u64x2::from([4, 5]); + let e = u64x2::from([2, 3]); + let r: u64x2 = unsafe { transmute(vbcaxq_u64(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vrax1q_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.rax1 + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let e = u64x2::from([4, 7]); + let r: u64x2 = unsafe { transmute(vrax1q_u64(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha3")] +unsafe fn test_vxarq_u64() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.xar + let a = u64x2::from([0, 1]); + let b = u64x2::from([2, 3]); + let e = u64x2::from([4, 4]); + let r: u64x2 = unsafe { transmute(vxarq_u64::<63>(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes")] fn test_vmull_p64() { @@ -698,6 +912,27 @@ fn main() { test_vsha512h2q_u64(); test_vsha512su0q_u64(); test_vsha512su1q_u64(); + + test_veor3q_s8(); + test_veor3q_s16(); + test_veor3q_s32(); + test_veor3q_s64(); + test_veor3q_u8(); + test_veor3q_u16(); + test_veor3q_u32(); + test_veor3q_u64(); + + test_vbcaxq_s8(); + test_vbcaxq_s16(); + test_vbcaxq_s32(); + test_vbcaxq_s64(); + test_vbcaxq_u8(); + test_vbcaxq_u16(); + test_vbcaxq_u32(); + test_vbcaxq_u64(); + + test_vrax1q_u64(); + test_vxarq_u64(); } test_vmull_p64(); diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index c322859fec2b3..147bfafba4f90 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -977,6 +977,88 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( ); } + "llvm.aarch64.crypto.eor3s.v2i64" + | "llvm.aarch64.crypto.eor3s.v4i32" + | "llvm.aarch64.crypto.eor3s.v8i16" + | "llvm.aarch64.crypto.eor3s.v16i8" + | "llvm.aarch64.crypto.eor3u.v2i64" + | "llvm.aarch64.crypto.eor3u.v4i32" + | "llvm.aarch64.crypto.eor3u.v8i16" + | "llvm.aarch64.crypto.eor3u.v16i8" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/EOR3--Three-way-exclusive-OR- + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + simd_trio_for_each_lane( + fx, + a, + b, + c, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane, c_lane| { + let xor = fx.bcx.ins().bxor(a_lane, b_lane); + fx.bcx.ins().bxor(xor, c_lane) + }, + ); + } + + "llvm.aarch64.crypto.bcaxs.v2i64" + | "llvm.aarch64.crypto.bcaxs.v4i32" + | "llvm.aarch64.crypto.bcaxs.v8i16" + | "llvm.aarch64.crypto.bcaxs.v16i8" + | "llvm.aarch64.crypto.bcaxu.v2i64" + | "llvm.aarch64.crypto.bcaxu.v4i32" + | "llvm.aarch64.crypto.bcaxu.v8i16" + | "llvm.aarch64.crypto.bcaxu.v16i8" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/BCAX--Bit-clear-and-exclusive-OR- + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + simd_trio_for_each_lane( + fx, + a, + b, + c, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane, c_lane| { + let band_not = fx.bcx.ins().band_not(b_lane, c_lane); + fx.bcx.ins().bxor(a_lane, band_not) + }, + ); + } + + "llvm.aarch64.crypto.rax1" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/RAX1--Rotate-and-exclusive-OR- + intrinsic_args!(fx, args => (a, b); intrinsic); + + simd_pair_for_each_lane( + fx, + a, + b, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| { + let rot = fx.bcx.ins().rotl_imm(b_lane, 1); + fx.bcx.ins().bxor(a_lane, rot) + }, + ); + } + + "llvm.aarch64.crypto.xar" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/XAR--Exclusive-OR-and-rotate- + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let c = c.load_scalar(fx); + + simd_pair_for_each_lane( + fx, + a, + b, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| { + let xor = fx.bcx.ins().bxor(a_lane, b_lane); + fx.bcx.ins().rotr(xor, c) + }, + ); + } + "llvm.aarch64.neon.pmull64" => { intrinsic_args!(fx, args => (a, b); intrinsic); From b2fee3072e5ead0a02c01198e8bb8b65c86ca818 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:31:47 +0200 Subject: [PATCH 06/51] Rustup to rustc 1.98.0-nightly (3daae5e42 2026-06-14) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index faadf08929557..15aeb1c18dd92 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-06-06" +channel = "nightly-2026-06-15" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From 07ed5e96789c325e1fdcb2126b8fa1e1e553f36a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:58:14 +0200 Subject: [PATCH 07/51] Fix rustc test suite --- scripts/test_rustc_tests.sh | 4 +++- src/base.rs | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 683adeb49edd1..98849b796328d 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -19,7 +19,7 @@ for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|//@ error-pa rm $test done -git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed +git checkout -- tests/ui/cross-crate/auxiliary/nested-struct-in-polymorphic-impl-method.rs # contains //~ERROR, but shouldn't be removed git checkout -- tests/ui/entry-point/auxiliary/bad_main_functions.rs # missing features @@ -149,6 +149,7 @@ rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift rm -r tests/run-make/short-ice # ICE backtrace begin/end marker mismatch rm -r tests/run-make/naked-dead-code-elimination # function not eliminated rm tests/ui/codegen/huge-stacks.rs # Cranelift doesn't allow stack frames to exceed 4GB +rm -r tests/run-make/doctests-test_harness # different thread names likely caused by -Zpanic-abort-tests # doesn't work due to the way the rustc test suite is invoked. # should work when using ./x.py test the way it is intended @@ -175,6 +176,7 @@ rm tests/codegen-units/item-collection/opaque-return-impls.rs # extra mono item. # ============ rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort +rm -r tests/run-make/staticlib-hide-internal-symbols # -Zstaticlib-hide-internal-symbols seems to be broken # bugs in the test suite # ====================== diff --git a/src/base.rs b/src/base.rs index 467eceea221c8..2b30b55318589 100644 --- a/src/base.rs +++ b/src/base.rs @@ -628,10 +628,11 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let ref_ = place.place_ref(fx, lval.layout()); lval.write_cvalue(fx, ref_); } - Rvalue::Reborrow(_, _, place) => { + Rvalue::Reborrow(ty, _, place) => { + assert_eq!(lval.layout().ty, ty); let cplace = codegen_place(fx, place); let val = cplace.to_cvalue(fx); - lval.write_cvalue(fx, val) + lval.write_cvalue_transmute(fx, val) } Rvalue::ThreadLocalRef(def_id) => { let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout()); From 59969ea6fd4a79f895c51368b9229cb7eb94bf4a Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Mon, 8 Jun 2026 21:44:31 +0200 Subject: [PATCH 08/51] remove LLVM `va_end` calls The operation is a no-op, so we skip it. --- src/intrinsics/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 31131bc889d57..e6b8c537d8451 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -1512,7 +1512,7 @@ fn codegen_regular_intrinsic_call<'tcx>( } // FIXME implement variadics in cranelift - sym::va_arg | sym::va_end => { + sym::va_arg => { fx.tcx.dcx().span_fatal( source_info.span, "Defining variadic functions is not yet supported by Cranelift", From bcdbc4ce599980a0796f9125e71fe17db054bcdc Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:25:34 +0200 Subject: [PATCH 09/51] Rustup to rustc 1.98.0-nightly (91fe22da8 2026-06-21) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 15aeb1c18dd92..48652dfac31f7 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-06-15" +channel = "nightly-2026-06-22" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From 84632b10e98571afee6ce2619ef92e4d02434a91 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:48:09 +0200 Subject: [PATCH 10/51] Fix rustc test suite --- scripts/test_rustc_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 98849b796328d..09046cdf8c135 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -177,12 +177,14 @@ rm tests/codegen-units/item-collection/opaque-return-impls.rs # extra mono item. rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort rm -r tests/run-make/staticlib-hide-internal-symbols # -Zstaticlib-hide-internal-symbols seems to be broken +rm -r tests/run-make/staticlib-rename-internal-symbols # needs files from staticlib-hide-internal-symbols # bugs in the test suite # ====================== rm tests/ui/process/nofile-limit.rs # FIXME some AArch64 linking issue rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables rm -r tests/incremental/extern_static/issue-49153.rs # assumes reference to undefined static gets optimized away +rm tests/ui/std/linux-getrandom-fallback.rs # missing needs-unwind rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # really slow with unoptimized libstd rm tests/ui/process/process-panic-after-fork.rs # same From 871eb58f72231a2c42687eeaf7b2f4371f7f7c1b Mon Sep 17 00:00:00 2001 From: Noah Lev Date: Thu, 18 Jun 2026 22:27:27 +0000 Subject: [PATCH 11/51] Extract all instance shim variants into new `ShimKind` enum They tend to have similar handling -- e.g., they should be the only input to the `mir_shims` query -- so it's cleaner to group them together. This will also make potential future refactorings easier, such as only carrying `GenericArgsRef` for instances that actually use it (e.g., `Item`) but not others (e.g., `CloneShim`). Many of the shim variants still have `Shim` at the end of their names. To make the refactoring easier and keep the diff clean, I will trim those suffixes off in the next commit. --- src/abi/mod.rs | 6 +++--- src/constant.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 0eb493036ce51..8f965a5ef7b12 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -16,9 +16,9 @@ use rustc_abi::{CanonAbi, ExternAbi, X86Call}; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::{ShimKind, TypeVisitableExt}; use rustc_session::Session; use rustc_span::Spanned; use rustc_target::callconv::{FnAbi, PassMode}; @@ -467,7 +467,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( } // We don't need AsyncDropGlueCtorShim here because it is not `noop func`, // it is `func returning noop future` - InstanceKind::DropGlue(_, None) => { + InstanceKind::Shim(ShimKind::DropGlue(_, None)) => { // empty drop glue - a nop. let dest = target.expect("Non terminating drop_in_place_real???"); let ret_block = fx.get_block(dest); @@ -725,7 +725,7 @@ pub(crate) fn codegen_drop<'tcx>( let ret_block = fx.get_block(target); // AsyncDropGlueCtorShim can't be here - if let ty::InstanceKind::DropGlue(_, None) = drop_instance.def { + if let ty::InstanceKind::Shim(ty::ShimKind::DropGlue(_, None)) = drop_instance.def { // we don't actually need to drop anything fx.bcx.ins().jump(ret_block, &[]); } else { diff --git a/src/constant.rs b/src/constant.rs index c986666f9c46e..a83bfd1cbad42 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -53,7 +53,7 @@ pub(crate) fn codegen_tls_ref<'tcx>( ) -> CValue<'tcx> { let tls_ptr = if !def_id.is_local() && fx.tcx.needs_thread_local_shim(def_id) { let instance = ty::Instance { - def: ty::InstanceKind::ThreadLocalShim(def_id), + def: ty::InstanceKind::Shim(ty::ShimKind::ThreadLocalShim(def_id)), args: ty::GenericArgs::empty(), }; let func_ref = fx.get_function_ref(instance); From 6adf588a8afae43e50dc99e414f8a123d6dd67ac Mon Sep 17 00:00:00 2001 From: Noah Lev Date: Thu, 18 Jun 2026 22:39:32 +0000 Subject: [PATCH 12/51] Strip vestigial `Shim` suffix from `ShimKind` variants --- src/constant.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/constant.rs b/src/constant.rs index a83bfd1cbad42..829e7d0dfb59c 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -53,7 +53,7 @@ pub(crate) fn codegen_tls_ref<'tcx>( ) -> CValue<'tcx> { let tls_ptr = if !def_id.is_local() && fx.tcx.needs_thread_local_shim(def_id) { let instance = ty::Instance { - def: ty::InstanceKind::Shim(ty::ShimKind::ThreadLocalShim(def_id)), + def: ty::InstanceKind::Shim(ty::ShimKind::ThreadLocal(def_id)), args: ty::GenericArgs::empty(), }; let func_ref = fx.get_function_ref(instance); From 604d40f04cfa1c0addda5c43e18b7e797dbec1ae Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 19 Jun 2026 16:33:06 +0200 Subject: [PATCH 13/51] Update to Cranelift 0.133 --- Cargo.lock | 73 +++++++++++++++++----------------- Cargo.toml | 24 +++++------ src/abi/mod.rs | 4 +- src/abi/pass_mode.rs | 8 +++- src/codegen_f16_f128.rs | 42 +++++++++++-------- src/common.rs | 5 ++- src/constant.rs | 2 +- src/inline_asm.rs | 4 +- src/intrinsics/llvm_aarch64.rs | 4 +- src/intrinsics/llvm_x86.rs | 26 +++++++----- src/intrinsics/mod.rs | 40 +++++++++++-------- src/intrinsics/simd.rs | 14 +++---- src/lib.rs | 2 +- src/optimize/peephole.rs | 24 ++++++----- src/pointer.rs | 9 ++++- src/value_and_place.rs | 9 ++--- src/vtable.rs | 4 +- 17 files changed, 162 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f793d6a08737..78abf73d36a5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,27 +43,27 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cranelift-assembler-x64" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c80cf55a351448317210f26c434be761bcb25e7b36116ec92f89540b73e2833" +checksum = "715783c05f20985a5dfe6bfdccbfcb146cb44bfd8f6ff1d09526c3bf442fdac5" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07937ca8617b340162fe3a4716be885b5847e9b56d6c7a89abbe4d42340fdc91" +checksum = "4c3da5a783f2b72af39ab98c1bf3157e081511e1febeafe550d71a4d0ba75f66" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88217b08180882436d54c0133274885c590698ae854e352bede1cda041230800" +checksum = "8e2e9b7adf77fa02204d4d523ae4f171b6591c2632b030865336335c7d7b420e" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -71,18 +71,18 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5c3cf7ba29fa56e56040848e34835d4e45988b2760ef212413409af95ffd8c1" +checksum = "90f09d9f397eae612ac15becf0e0b2165d2232003f4f2a1549572a724d1c48c5" dependencies = [ "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe1aac2efd4cba2047845fce38a68519935a30e20c8a6294ba7e2f448fe722d" +checksum = "5e004cf1270abc82f7b9fb32d1a9d73a1cc0d5a0215b97db8c32658748e78b01" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0909eaf9d6f18f5bf802d50608cb4368ac340fbd03cc44f2888d1cfcc3faa64e" +checksum = "04f32034641f96b123e4fdb5666e726d9f252e222638fc8fdd905b4ff18c3c4e" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -119,24 +119,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a8da8be283f49cda7d0ef228c94f10d791e517b27b0c7e282dadd2e79ce45" +checksum = "746566ae868b0e87a89206b3856350886a4fc2e078ce6485bec2ea6727c31685" [[package]] name = "cranelift-control" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5b19c81145146da1f7afda2e7f52111842fe6793512e740ad5cf3f5639e6212" +checksum = "def01ab5cd08a1be551d4bc96adc6af91f89138c4f81d0a60fdf3b9f82272703" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a55309b47e6633ab05821304206cb1e92952e845b1224985562bb7ac1e92323" +checksum = "341d5e1e071320505ebbc8194a8eb61fa94394b1ed93ba3cbec3be5fa1c3c1ce" dependencies = [ "cranelift-bitset", "wasmtime-internal-core", @@ -144,11 +144,12 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064d2d3533d9608f1cf44c8899cf2f7f33feb70300b0fb83e687b0d9e7b91147" +checksum = "97c6f3e2419ecb54a5503994d35421554c5e487d0493bc86ea96907bab51fd29" dependencies = [ "cranelift-codegen", + "hashbrown 0.17.0", "log", "smallvec", "target-lexicon", @@ -156,15 +157,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac4e0bc095b2dab2212d1e99d7a74b62afc1485db023f1c0cb34a68758f7bd1" +checksum = "21aa47a5e0b1e9fb2c9348459088d5dbb872ebe17781ada1270d8364c7e73257" [[package]] name = "cranelift-jit" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b48c2a0720c7d62aadd508c662b9bf666b614a47a888589e553e0511620635e" +checksum = "0401e1c72d772c397726f49b29f048fe397fe6b1f761283c4bfa1f9ebe6cb86b" dependencies = [ "anyhow", "cranelift-codegen", @@ -183,9 +184,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28f05d9efce7a4e8c2ceec49c76d26e53f1ee8cb13de822b6ca5118d48f50976" +checksum = "5a1992fd5ae19ac2a8e8f37b134a797a7a3bde9ee9d3117d560ad382b985dbef" dependencies = [ "anyhow", "cranelift-codegen", @@ -194,9 +195,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a40053f5cb925451dd1d57393d14ad3145c8e0786701c27b5415ebb9a3ba4f" +checksum = "c3054a03ac285b170662ec9530602b01cd394a7428cd753b48d9cae51f05249a" dependencies = [ "cranelift-codegen", "libc", @@ -205,9 +206,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7a263727954f7b310796e1b5543e6dfd6afed7e15c62f2454b51b6f38a39e1" +checksum = "b2adff988bc0579a7233517ce0a9276c61cca2ec7e2e56e83226a5b32e35653b" dependencies = [ "anyhow", "cranelift-codegen", @@ -220,9 +221,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.132.0" +version = "0.133.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ceab9a53f7d362c89841fbaa8e63e44d47c40e91dc96ee6f777fca5d6b323b" +checksum = "fd2498406acfcfd39c69078af187154ac8dcd31d3082888b258deb2a4fd0ecf4" [[package]] name = "crc32fast" @@ -492,9 +493,9 @@ checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "wasmtime-internal-core" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bdae4b55b15a23d774b15f6e7cd90ae0d0aa17c47c12b4db098b3dd11ba9d58" +checksum = "4f6ce74d60a8ed870548e7efa9710c54f982bbfcf80e5ee5eee8498318616483" dependencies = [ "hashbrown 0.17.0", "libm", @@ -502,9 +503,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a312ba8bb77955dcd44294a223e7f124c3071ff966583d385d3f6a4639c62e3" +checksum = "0bba33d9d951a9a974a866e80c864a9a46b28086e369582e0caa78e14a9f29e4" dependencies = [ "cfg-if", "libc", diff --git a/Cargo.toml b/Cargo.toml index 17ddbada21646..aa43e43d2165a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,12 +8,12 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.132.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } -cranelift-frontend = { version = "0.132.0" } -cranelift-module = { version = "0.132.0" } -cranelift-native = { version = "0.132.0" } -cranelift-jit = { version = "0.132.0", optional = true } -cranelift-object = { version = "0.132.0" } +cranelift-codegen = { version = "0.133.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } +cranelift-frontend = { version = "0.133.0" } +cranelift-module = { version = "0.133.0" } +cranelift-native = { version = "0.133.0" } +cranelift-jit = { version = "0.133.0", optional = true } +cranelift-object = { version = "0.133.0", default-features = false } target-lexicon = "0.13" gimli = { version = "0.33", default-features = false, features = ["write"] } object = { version = "0.39.1", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } @@ -24,12 +24,12 @@ smallvec = "1.8.1" # Uncomment to use an unreleased version of cranelift #[patch.crates-io] -#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-45.0.0" } -#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-45.0.0" } -#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-45.0.0" } -#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-45.0.0" } -#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-45.0.0" } -#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-45.0.0" } +#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } +#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } +#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } +#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } +#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } +#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } # Uncomment to use local checkout of cranelift #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 0eb493036ce51..5c1a3e033acaa 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -147,7 +147,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { .map(|(param, &arg)| { if param.value_type == types::I128 { let arg_ptr = self.create_stack_slot(16, 16); - arg_ptr.store(self, arg, MemFlags::trusted()); + arg_ptr.store(self, arg, MemFlagsData::trusted()); (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self)) } else { (param, arg) @@ -179,7 +179,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { self.lib_call_unadjusted(name, params, vec![], &args); - Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]) + Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlagsData::trusted())]) } else { Cow::Borrowed(self.lib_call_unadjusted(name, params, returns, &args)) } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 612f89e6a4217..3d655eb18d7d9 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -211,7 +211,11 @@ pub(super) fn to_casted_value<'tcx>( cast_target_to_abi_params(cast) .into_iter() .map(|(offset, param)| { - ptr.offset_i64(fx, offset.bytes() as i64).load(fx, param.value_type, MemFlags::new()) + ptr.offset_i64(fx, offset.bytes() as i64).load( + fx, + param.value_type, + MemFlagsData::new(), + ) }) .collect() } @@ -237,7 +241,7 @@ pub(super) fn from_casted_value<'tcx>( ptr.offset_i64(fx, offset.bytes() as i64).store( fx, block_params_iter.next().unwrap(), - MemFlags::new(), + MemFlagsData::trusted(), ) } assert_eq!(block_params_iter.next(), None, "Leftover block param"); diff --git a/src/codegen_f16_f128.rs b/src/codegen_f16_f128.rs index e63e4bebf78a2..7a386db0cd6a3 100644 --- a/src/codegen_f16_f128.rs +++ b/src/codegen_f16_f128.rs @@ -7,7 +7,7 @@ pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value let (value, arg_ty) = if fx.tcx.sess.target.is_like_darwin && fx.tcx.sess.target.arch == Arch::X86_64 { ( - fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value), + fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value), lib_call_arg_param(fx.tcx, types::I16, false), ) } else { @@ -33,7 +33,11 @@ pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value vec![AbiParam::new(ret_ty)], &[value], )[0]; - if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } + if ret_ty == types::I16 { + fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), ret) + } else { + ret + } } fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { @@ -48,7 +52,11 @@ fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { vec![AbiParam::new(ret_ty)], &[value], )[0]; - if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } + if ret_ty == types::I16 { + fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), ret) + } else { + ret + } } // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have @@ -139,52 +147,52 @@ pub(crate) fn codegen_f128_binop( } pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { - let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); + let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value); let bits = fx.bcx.ins().bxor_imm(bits, 0x8000); - fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) + fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits) } pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { - let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); + let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value); let (low, high) = fx.bcx.ins().isplit(bits); let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64); let bits = fx.bcx.ins().iconcat(low, high); - fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) + fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) } pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { - let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); + let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value); let bits = fx.bcx.ins().band_imm(bits, 0x7fff); - fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) + fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits) } pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { - let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); + let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value); let (low, high) = fx.bcx.ins().isplit(bits); let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64); let bits = fx.bcx.ins().iconcat(low, high); - fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) + fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) } pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { - let lhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), lhs); - let rhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), rhs); + let lhs = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), lhs); + let rhs = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), rhs); let res = fx.bcx.ins().band_imm(lhs, 0x7fff); let sign = fx.bcx.ins().band_imm(rhs, 0x8000); let res = fx.bcx.ins().bor(res, sign); - fx.bcx.ins().bitcast(types::F16, MemFlags::new(), res) + fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), res) } pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { - let lhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), lhs); - let rhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), rhs); + let lhs = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), lhs); + let rhs = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), rhs); let (low, lhs_high) = fx.bcx.ins().isplit(lhs); let (_, rhs_high) = fx.bcx.ins().isplit(rhs); let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64); let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64); let high = fx.bcx.ins().bor(high, sign); let res = fx.bcx.ins().iconcat(low, high); - fx.bcx.ins().bitcast(types::F128, MemFlags::new(), res) + fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), res) } pub(crate) fn codegen_cast( diff --git a/src/common.rs b/src/common.rs index b22afca847aa9..a49fda28b18fb 100644 --- a/src/common.rs +++ b/src/common.rs @@ -146,7 +146,7 @@ pub(crate) fn codegen_icmp_imm( } pub(crate) fn codegen_bitcast(fx: &mut FunctionCx<'_, '_, '_>, dst_ty: Type, val: Value) -> Value { - let mut flags = MemFlags::new(); + let mut flags = MemFlagsData::new(); flags.set_endianness(match fx.tcx.data_layout.endian { rustc_abi::Endian::Big => cranelift_codegen::ir::Endianness::Big, rustc_abi::Endian::Little => cranelift_codegen::ir::Endianness::Little, @@ -401,7 +401,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { }); let base_ptr = self.bcx.ins().stack_addr(self.pointer_type, stack_slot, 0); let misalign_offset = self.bcx.ins().band_imm(base_ptr, i64::from(align - 1)); - let realign_offset = self.bcx.ins().irsub_imm(misalign_offset, i64::from(align)); + let align = self.bcx.ins().iconst(self.pointer_type, i64::from(align)); + let realign_offset = self.bcx.ins().isub(align, misalign_offset); Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset)) } } diff --git a/src/constant.rs b/src/constant.rs index c986666f9c46e..7c9d8a8e64394 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -131,7 +131,7 @@ pub(crate) fn codegen_const_value<'tcx>( // FIXME avoid this extra copy to the stack and directly write to the final // destination let place = CPlace::new_stack_slot(fx, layout); - place.to_ptr().store(fx, val, MemFlags::trusted()); + place.to_ptr().store(fx, val, MemFlagsData::trusted()); place.to_cvalue(fx) } } diff --git a/src/inline_asm.rs b/src/inline_asm.rs index f9fc7002be87b..d531f496eb038 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -847,7 +847,7 @@ fn call_inline_asm<'tcx>( stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store( fx, value, - MemFlags::trusted(), + MemFlagsData::trusted(), ); } @@ -865,7 +865,7 @@ fn call_inline_asm<'tcx>( let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load( fx, ty, - MemFlags::trusted(), + MemFlagsData::trusted(), ); place.write_cvalue(fx, CValue::by_val(value, place.layout())); } diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index c322859fec2b3..4737061e03f28 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -426,7 +426,7 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( let t_idx = fx.bcx.ins().uextend(fx.pointer_type, idx_lane); let t_lane = t.value_lane_dyn(fx, t_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, t_lane); - ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlagsData::trusted()); } } "llvm.aarch64.neon.tbl1.v16i8" => { @@ -440,7 +440,7 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( let t_idx = fx.bcx.ins().uextend(fx.pointer_type, idx_lane); let t_lane = t.value_lane_dyn(fx, t_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, t_lane); - ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlagsData::trusted()); } } diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs index 61f48fa977435..29303f701588e 100644 --- a/src/intrinsics/llvm_x86.rs +++ b/src/intrinsics/llvm_x86.rs @@ -117,8 +117,11 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let src_lane = src.value_lane(fx, lane_idx).load_scalar(fx); let index_lane = index.value_lane(fx, lane_idx).load_scalar(fx); let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); - let mask_lane = - fx.bcx.ins().bitcast(mask_lane_clif_ty.as_int(), MemFlags::new(), mask_lane); + let mask_lane = fx.bcx.ins().bitcast( + mask_lane_clif_ty.as_int(), + MemFlagsData::new(), + mask_lane, + ); let if_enabled = fx.bcx.create_block(); let if_disabled = fx.bcx.create_block(); @@ -146,7 +149,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( }; let offset = fx.bcx.ins().imul(index_lane, scale); let lane_ptr = fx.bcx.ins().iadd(ptr, offset); - let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), lane_ptr, 0); + let res = fx.bcx.ins().load(lane_clif_ty, MemFlagsData::trusted(), lane_ptr, 0); fx.bcx.ins().jump(next, &[res.into()]); fx.bcx.switch_to_block(if_disabled); @@ -163,7 +166,8 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( for lane_idx in std::cmp::min(src_lane_count, index_lane_count)..ret_lane_count { let zero_lane = fx.bcx.ins().iconst(mask_lane_clif_ty.as_int(), 0); - let zero_lane = fx.bcx.ins().bitcast(mask_lane_clif_ty, MemFlags::new(), zero_lane); + let zero_lane = + fx.bcx.ins().bitcast(mask_lane_clif_ty, MemFlagsData::new(), zero_lane); ret.place_lane(fx, lane_idx) .write_cvalue(fx, CValue::by_val(zero_lane, ret_lane_layout)); } @@ -325,7 +329,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, a_lane); - ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlagsData::trusted()); } if intrinsic == "llvm.x86.avx2.pshuf.b" { @@ -337,7 +341,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, a_lane); - ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlagsData::trusted()); } } } @@ -352,7 +356,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( ret.place_typed_lane(fx, fx.tcx.types.u32, j).to_ptr().store( fx, value, - MemFlags::trusted(), + MemFlagsData::trusted(), ); } } @@ -406,12 +410,12 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( ret.place_typed_lane(fx, fx.tcx.types.u128, 0).to_ptr().store( fx, res_low, - MemFlags::trusted(), + MemFlagsData::trusted(), ); ret.place_typed_lane(fx, fx.tcx.types.u128, 1).to_ptr().store( fx, res_high, - MemFlags::trusted(), + MemFlagsData::trusted(), ); } "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => { @@ -465,7 +469,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); - Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlags::trusted()); + Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlagsData::trusted()); ret.write_cvalue(fx, CValue::by_val(cb_out, fx.layout_of(fx.tcx.types.u8))); } "llvm.x86.subborrow.32" | "llvm.x86.subborrow.64" => { @@ -502,7 +506,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( "llvm.x86.sse2.psra.w" => { intrinsic_args!(fx, args => (a, count); intrinsic); - let count_lane = count.force_stack(fx).0.load(fx, types::I64, MemFlags::trusted()); + let count_lane = count.force_stack(fx).0.load(fx, types::I64, MemFlagsData::trusted()); let lane_ty = fx.clif_type(a.layout().ty.simd_size_and_type(fx.tcx).1).unwrap(); let max_count = fx.bcx.ins().iconst(types::I64, i64::from(lane_ty.bits() - 1)); let saturated_count = fx.bcx.ins().umin(count_lane, max_count); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index e6b8c537d8451..f7cfc822293f3 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -875,7 +875,7 @@ fn codegen_regular_intrinsic_call<'tcx>( } let clif_ty = fx.clif_type(ty).unwrap(); - let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr); + let val = fx.bcx.ins().atomic_load(clif_ty, MemFlagsData::trusted(), ptr); let val = CValue::by_val(val, fx.layout_of(ty)); ret.write_cvalue(fx, val); @@ -911,7 +911,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = val.load_scalar(fx); - fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); + fx.bcx.ins().atomic_store(MemFlagsData::trusted(), val, ptr); } sym::atomic_xchg => { intrinsic_args!(fx, args => (ptr, new); intrinsic); @@ -929,7 +929,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let new = new.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Xchg, ptr, new); let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); @@ -950,7 +951,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let test_old = test_old.load_scalar(fx); let new = new.load_scalar(fx); - let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new); + let old = fx.bcx.ins().atomic_cas(MemFlagsData::trusted(), ptr, test_old, new); let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old); let ret_val = CValue::by_val_pair(old, is_eq, ret.layout()); @@ -974,7 +975,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let amount = amount.load_scalar(fx); let old = - fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount); + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Add, ptr, amount); let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); @@ -996,7 +997,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let amount = amount.load_scalar(fx); let old = - fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount); + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Sub, ptr, amount); let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); @@ -1017,7 +1018,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::And, ptr, src); let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); @@ -1038,7 +1040,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Or, ptr, src); let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); @@ -1059,7 +1062,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Xor, ptr, src); let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); @@ -1080,7 +1084,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Nand, ptr, src); let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); @@ -1101,7 +1106,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Smax, ptr, src); let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); @@ -1122,7 +1128,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Umax, ptr, src); let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); @@ -1143,7 +1150,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Smin, ptr, src); let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); @@ -1164,7 +1172,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let src = src.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlagsData::trusted(), AtomicRmwOp::Umin, ptr, src); let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); @@ -1471,8 +1480,7 @@ fn codegen_regular_intrinsic_call<'tcx>( fx.bcx.ins().iconst(types::I8, 1) } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) { // Can't use `trusted` for these loads; they could be unaligned. - let mut flags = MemFlags::new(); - flags.set_notrap(); + let flags = MemFlagsData::new().with_notrap(); let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0); let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0); fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val) diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 5d7e457b8e1ba..e118447df4810 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -850,7 +850,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( m.force_stack(fx).0.load( fx, Type::int(expected_int_bits as u16).unwrap(), - MemFlags::trusted(), + MemFlagsData::trusted(), ) } _ => { @@ -1026,8 +1026,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( generic_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(); let memflags = match alignment { - SimdAlign::Unaligned => MemFlags::new().with_notrap(), - _ => MemFlags::trusted(), + SimdAlign::Unaligned => MemFlagsData::new().with_notrap(), + _ => MemFlagsData::trusted(), }; for lane_idx in 0..val_lane_count { @@ -1081,7 +1081,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.seal_block(if_disabled); fx.bcx.switch_to_block(if_enabled); - let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0); + let res = fx.bcx.ins().load(lane_clif_ty, MemFlagsData::trusted(), ptr_lane, 0); fx.bcx.ins().jump(next, &[res.into()]); fx.bcx.switch_to_block(if_disabled); @@ -1114,8 +1114,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( generic_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(); let memflags = match alignment { - SimdAlign::Unaligned => MemFlags::new().with_notrap(), - _ => MemFlags::trusted(), + SimdAlign::Unaligned => MemFlagsData::new().with_notrap(), + _ => MemFlagsData::trusted(), }; for lane_idx in 0..ret_lane_count { @@ -1170,7 +1170,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.seal_block(if_enabled); fx.bcx.switch_to_block(if_enabled); - fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_lane, 0); + fx.bcx.ins().store(MemFlagsData::trusted(), val_lane, ptr_lane, 0); fx.bcx.ins().jump(next, &[]); fx.bcx.seal_block(next); diff --git a/src/lib.rs b/src/lib.rs index af783f31a2136..b506fdc84d7a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,7 +83,7 @@ mod prelude { pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; pub(crate) use cranelift_codegen::ir::function::Function; pub(crate) use cranelift_codegen::ir::{ - AbiParam, Block, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, StackSlot, + AbiParam, Block, FuncRef, Inst, InstBuilder, MemFlagsData, Signature, SourceLoc, StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value, types, }; pub(crate) use cranelift_module::{self, DataDescription, FuncId, Linkage, Module}; diff --git a/src/optimize/peephole.rs b/src/optimize/peephole.rs index f38c1f96e6ed5..fe301cb85cf75 100644 --- a/src/optimize/peephole.rs +++ b/src/optimize/peephole.rs @@ -7,17 +7,19 @@ use cranelift_frontend::FunctionBuilder; /// If the given value was produced by the lowering of `Rvalue::Not` return the input and true, /// otherwise return the given value and false. pub(crate) fn maybe_unwrap_bool_not(bcx: &mut FunctionBuilder<'_>, arg: Value) -> (Value, bool) { - if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { - match bcx.func.dfg.insts[arg_inst] { - // This is the lowering of `Rvalue::Not` - InstructionData::IntCompareImm { - opcode: Opcode::IcmpImm, - cond: IntCC::Equal, - arg, - imm, - } if imm.bits() == 0 => (arg, true), - _ => (arg, false), - } + if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) + && let InstructionData::IntCompare { + opcode: Opcode::Icmp, + cond: IntCC::Equal, + args: [icmp_arg, imm], + } = bcx.func.dfg.insts[arg_inst] + && let ValueDef::Result(imm_inst, 0) = bcx.func.dfg.value_def(imm) + && let InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } = + bcx.func.dfg.insts[imm_inst] + && imm.bits() == 0 + { + // This is the lowering of `Rvalue::Not` + (icmp_arg, true) } else { (arg, false) } diff --git a/src/pointer.rs b/src/pointer.rs index 2750caa216e6b..56ea1096c770b 100644 --- a/src/pointer.rs +++ b/src/pointer.rs @@ -106,7 +106,12 @@ impl Pointer { } } - pub(crate) fn load(self, fx: &mut FunctionCx<'_, '_, '_>, ty: Type, flags: MemFlags) -> Value { + pub(crate) fn load( + self, + fx: &mut FunctionCx<'_, '_, '_>, + ty: Type, + flags: MemFlagsData, + ) -> Value { match self.base { PointerBase::Addr(base_addr) => fx.bcx.ins().load(ty, flags, base_addr, self.offset), PointerBase::Stack(stack_slot) => fx.bcx.ins().stack_load(ty, stack_slot, self.offset), @@ -114,7 +119,7 @@ impl Pointer { } } - pub(crate) fn store(self, fx: &mut FunctionCx<'_, '_, '_>, value: Value, flags: MemFlags) { + pub(crate) fn store(self, fx: &mut FunctionCx<'_, '_, '_>, value: Value, flags: MemFlagsData) { match self.base { PointerBase::Addr(base_addr) => { fx.bcx.ins().store(flags, value, base_addr, self.offset); diff --git a/src/value_and_place.rs b/src/value_and_place.rs index a2a2cac3faaa8..09b51a47b2a70 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -143,8 +143,7 @@ impl<'tcx> CValue<'tcx> { } _ => unreachable!("{:?}", layout.ty), }; - let mut flags = MemFlags::new(); - flags.set_notrap(); + let flags = MemFlagsData::new().with_notrap(); ptr.load(fx, clif_ty, flags) } CValueInner::ByVal(value) => value, @@ -166,8 +165,7 @@ impl<'tcx> CValue<'tcx> { let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar); let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar); - let mut flags = MemFlags::new(); - flags.set_notrap(); + let flags = MemFlagsData::new().with_notrap(); let val1 = ptr.load(fx, clif_ty1, flags); let val2 = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags); (val1, val2) @@ -599,8 +597,7 @@ impl<'tcx> CPlace<'tcx> { return; } - let mut flags = MemFlags::new(); - flags.set_notrap(); + let flags = MemFlagsData::new().with_notrap(); match from.0 { CValueInner::ByVal(val) => { diff --git a/src/vtable.rs b/src/vtable.rs index b5d241d8f39f2..476138909b5ed 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -5,8 +5,8 @@ use crate::constant::data_id_for_vtable; use crate::prelude::*; -pub(crate) fn vtable_memflags() -> MemFlags { - let mut flags = MemFlags::trusted(); // A vtable access is always aligned and will never trap. +pub(crate) fn vtable_memflags() -> MemFlagsData { + let mut flags = MemFlagsData::trusted(); // A vtable access is always aligned and will never trap. flags.set_readonly(); // A vtable is always read-only. flags } From 052d9588fff37cf31c7c6822561dca85db88f074 Mon Sep 17 00:00:00 2001 From: Rudraksh Joshi <127816064+0xmuon@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:17:49 +0530 Subject: [PATCH 14/51] ci: monthly Cranelift release branch automation regex (#1665) --- .github/workflows/cranelift-release-branch.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cranelift-release-branch.yml b/.github/workflows/cranelift-release-branch.yml index 5cfdc3e8f2936..bd268e7f1762b 100644 --- a/.github/workflows/cranelift-release-branch.yml +++ b/.github/workflows/cranelift-release-branch.yml @@ -38,6 +38,8 @@ jobs: - name: Patch Cargo.toml to use release branch Cranelift run: | + sed -i -E '/cranelift-/s/"0\.[0-9]+\.0"/"*"/g' Cargo.toml + cat >>Cargo.toml < Date: Fri, 22 May 2026 09:47:15 +0800 Subject: [PATCH 15/51] add IsRigid marker to TyKind::Alias and constants --- src/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common.rs b/src/common.rs index b22afca847aa9..69e472b33abac 100644 --- a/src/common.rs +++ b/src/common.rs @@ -349,7 +349,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { self.instance.instantiate_mir_and_normalize_erasing_regions( self.tcx, ty::TypingEnv::fully_monomorphized(), - ty::EarlyBinder::bind(value), + ty::EarlyBinder::bind(self.tcx, value), ) } From 316b36fcb330ae4c9df6e37c34efb2112e2985c3 Mon Sep 17 00:00:00 2001 From: Cathal Mullan Date: Thu, 25 Jun 2026 13:46:44 +0100 Subject: [PATCH 16/51] Bump `graviola` to v0.4.1 --- build_system/tests.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/build_system/tests.rs b/build_system/tests.rs index 685bf8ce9a891..bc7c8d1d5d1f3 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -146,9 +146,9 @@ static REGEX: CargoProject = CargoProject::new(REGEX_REPO.source_dir(), "regex_t pub(crate) static GRAVIOLA_REPO: GitRepo = GitRepo::github( "ctz", "graviola", - "c779b83cfd7114c4802293700c92cfb5e05cb4b7", + "7763d0cc617d6f5f66c3bc0fe9b3d8581d781b6a", // v0.4.1 &["thirdparty/cavp", "thirdparty/wycheproof"], - "e0925ceb21a56101", + "7fa5a75b9fb1ac40", "graviola", ); @@ -231,16 +231,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512bw", "1"); test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512vl", "1"); - test_cmd.args([ - "-p", - "graviola", - "--lib", - "--", - "-q", - // FIXME: Disable AVX-512 until intrinsics are supported. - "--skip", - "check_counter512", - ]); + test_cmd.args(["-p", "graviola", "--lib", "--", "-q"]); spawn_and_wait(test_cmd); } else { eprintln!("Cross-Compiling: Not running tests"); From 014bb13f7be7cec69f8d05ee0a58f7e0da7b67ef Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 12 Jun 2026 14:04:35 +0200 Subject: [PATCH 17/51] Impl MIR lowering side-tables for #[splat] --- src/abi/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 8f965a5ef7b12..012951098fa15 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -271,6 +271,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ .map(|local| { let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty); + // FIXME(splat): un-tuple splatted arguments in codegen, for performance // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482 if Some(local) == fx.mir.spread_arg { // This argument (e.g. the last argument in the "rust-call" ABI) From cd00d477b20767e162cecafa064fbd75643ac393 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 27 Jun 2026 23:14:21 +0200 Subject: [PATCH 18/51] use `"llvm.prefetch.p0"` instead of `"llvm.prefetch"` LLVM updated the name, the old one still works but stdarch is now using the new one --- src/intrinsics/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 2dee9176936fd..9ec84d135dc8f 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -19,7 +19,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( } match intrinsic { - "llvm.prefetch" => { + "llvm.prefetch.p0" => { // Nothing to do. This is merely a perf hint. } From f079d012067c51f9e47297ef1da8b9ce859c065b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:51:02 +0200 Subject: [PATCH 19/51] Remove trailing whitespace --- .github/workflows/cranelift-release-branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cranelift-release-branch.yml b/.github/workflows/cranelift-release-branch.yml index bd268e7f1762b..b10f8493a3fb6 100644 --- a/.github/workflows/cranelift-release-branch.yml +++ b/.github/workflows/cranelift-release-branch.yml @@ -49,7 +49,7 @@ jobs: cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "$(echo $WASMTIME_RELEASE_BRANCH)" } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "$(echo $WASMTIME_RELEASE_BRANCH)" } EOF - + cargo update env: WASMTIME_RELEASE_BRANCH: ${{ steps.wasmtime_release_branch.outputs.branch }} From c2f57f22c19385cad875054cd8173242d546b0cc Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:28:12 +0200 Subject: [PATCH 20/51] Rustup to rustc 1.98.0-nightly (df6ee909e 2026-06-28) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 48652dfac31f7..ba32d5fd58d5e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-06-22" +channel = "nightly-2026-06-29" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From c232e6289b7757783926a9970618814948f9457d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:18:08 +0200 Subject: [PATCH 21/51] Minor formatting change --- src/driver/aot.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index fcaf80d968a49..f4acfff8bb9fe 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -308,11 +308,8 @@ impl ExtraBackendMethods for AotDriver { impl WriteBackendMethods for AotDriver { type Module = AotModule; - - type TargetMachine = (); - type ModuleBuffer = Infallible; - + type TargetMachine = (); type ThinData = Infallible; fn target_machine_factory( From b25de25b97056a64af6243b7f7b97b977066267f Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sun, 28 Jun 2026 22:01:18 -0700 Subject: [PATCH 22/51] Rename `align` to `platform_align` on `Scalar` and `Primitive` To emphasize that just because you see a `Scalar(I32)` that doesn't really tell you anything about the alignment it has -- one should be looking at the type (well, the place) for that. No actual layout or behaviour changes in *this* PR. --- src/value_and_place.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/value_and_place.rs b/src/value_and_place.rs index a2a2cac3faaa8..995a70f24240b 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -56,7 +56,7 @@ fn codegen_field<'tcx>( } fn scalar_pair_calculate_b_offset(tcx: TyCtxt<'_>, a_scalar: Scalar, b_scalar: Scalar) -> Offset32 { - let b_offset = a_scalar.size(&tcx).align_to(b_scalar.align(&tcx).abi); + let b_offset = a_scalar.size(&tcx).align_to(b_scalar.default_align(&tcx).abi); Offset32::new(b_offset.bytes().try_into().unwrap()) } From cbff8884e3bc185f0e2142a660d6aecfbab0022f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:49:06 +0200 Subject: [PATCH 23/51] Better panic messages --- src/base.rs | 33 ++++++++++++++++++--------------- src/pretty_clif.rs | 28 ++++++++++++++++++---------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/base.rs b/src/base.rs index 2b30b55318589..6c6122501c4a0 100644 --- a/src/base.rs +++ b/src/base.rs @@ -18,7 +18,7 @@ use rustc_span::Symbol; use crate::constant::ConstantCx; use crate::debuginfo::{FunctionDebugContext, TypeDebugContext}; use crate::prelude::*; -use crate::pretty_clif::CommentWriter; +use crate::pretty_clif::{CommentWriter, format_clif_ir_header}; use crate::{codegen_f16_f128, enable_verifier}; pub(crate) struct CodegenedFunction { @@ -168,23 +168,15 @@ pub(crate) fn compile_fn( #[cfg(false)] let _clif_guard = { - use std::fmt::Write; - + let isa = module.isa(); + let symbol_name = &codegened_func.symbol_name; let func_clone = context.func.clone(); let clif_comments_clone = clif_comments.clone(); - let mut clif = String::new(); - for flag in module.isa().flags().iter() { - writeln!(clif, "set {}", flag).unwrap(); - } - write!(clif, "target {}", module.isa().triple().architecture.to_string()).unwrap(); - for isa_flag in module.isa().isa_flags().iter() { - write!(clif, " {}", isa_flag).unwrap(); - } - writeln!(clif, "\n").unwrap(); - writeln!(clif, "; symbol {}", codegened_func.symbol_name).unwrap(); + + let clif = crate::pretty_clif::format_clif_ir_header(isa, symbol_name); crate::PrintOnPanic(move || { let mut clif = clif.clone(); - ::cranelift_codegen::write::decorate_function( + cranelift_codegen::write::decorate_function( &mut &clif_comments_clone, &mut clif, &func_clone, @@ -221,7 +213,18 @@ pub(crate) fn compile_fn( early_dcx.early_fatal(format!("cranelift verify error:\n{}", pretty_error)); } Err(err) => { - panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name); + let mut clif = format_clif_ir_header(module.isa(), &codegened_func.symbol_name); + cranelift_codegen::write::decorate_function( + &mut &clif_comments, + &mut clif, + &context.func, + ) + .unwrap(); + + panic!( + "Error while defining {name}: {err:?}\n\nPost-optimization Cranelift IR:\n{clif}", + name = codegened_func.symbol_name + ); } } }); diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 918fe3d2a3895..fee3562c7bede 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -61,6 +61,7 @@ use std::io::Write; use cranelift_codegen::entity::SecondaryMap; use cranelift_codegen::ir::entities::AnyEntity; +use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::write::{FuncWriter, PlainWriter}; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_session::Session; @@ -242,6 +243,22 @@ impl FunctionCx<'_, '_, '_> { } } +pub(crate) fn format_clif_ir_header(isa: &dyn TargetIsa, symbol_name: &str) -> String { + use std::fmt::Write; + + let mut header = String::new(); + for flag in isa.flags().iter() { + writeln!(header, "set {}", flag).unwrap(); + } + write!(header, "target {}", isa.triple().architecture.to_string()).unwrap(); + for isa_flag in isa.isa_flags().iter() { + write!(header, " {}", isa_flag).unwrap(); + } + writeln!(header, "\n").unwrap(); + writeln!(header, "; symbol {}", symbol_name).unwrap(); + header +} + pub(crate) fn should_write_ir(sess: &Session) -> bool { sess.opts.output_types.contains_key(&OutputType::LlvmAssembly) } @@ -280,18 +297,9 @@ pub(crate) fn write_clif_file( ) { // FIXME work around filename too long errors write_ir_file(output_filenames, &format!("{}.{}.clif", symbol_name, postfix), |file| { - let mut clif = String::new(); + let mut clif = format_clif_ir_header(isa, symbol_name); cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func).unwrap(); - for flag in isa.flags().iter() { - writeln!(file, "set {}", flag)?; - } - write!(file, "target {}", isa.triple().architecture)?; - for isa_flag in isa.isa_flags().iter() { - write!(file, " {}", isa_flag)?; - } - writeln!(file, "\n")?; - writeln!(file)?; file.write_all(clif.as_bytes())?; Ok(()) }); From e9ad5d97a5a25f48eb1b169ff1dc8a798c509486 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:58:11 +0200 Subject: [PATCH 24/51] Replace keep_sysroot config with --keep-sysroot cli flag This makes it less likely to forget that you have it set. --- build_system/abi_cafe.rs | 2 ++ build_system/build_sysroot.rs | 19 ++++++++++++++----- build_system/main.rs | 6 ++++++ build_system/tests.rs | 3 +++ build_system/usage.txt | 5 +++++ config.txt | 7 ------- 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index e90e7ccf4ff84..8db09f08063a8 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -21,6 +21,7 @@ pub(crate) fn run( rustup_toolchain_name: Option<&str>, bootstrap_host_compiler: &Compiler, panic_unwind_support: bool, + keep_sysroot: bool, ) { std::fs::create_dir_all(&dirs.download_dir).unwrap(); ABI_CAFE_REPO.fetch(dirs); @@ -35,6 +36,7 @@ pub(crate) fn run( rustup_toolchain_name, bootstrap_host_compiler.triple.clone(), panic_unwind_support, + keep_sysroot, ); eprintln!("Running abi-cafe"); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 5c8b719ad5404..0cee2a975d896 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -8,7 +8,7 @@ use crate::rustc_info::{get_default_sysroot, get_file_name}; use crate::utils::{ CargoProject, Compiler, LogGroup, ensure_empty_dir, spawn_and_wait, try_hard_link, }; -use crate::{CodegenBackend, SysrootKind, config}; +use crate::{CodegenBackend, SysrootKind}; pub(crate) fn build_sysroot( dirs: &Dirs, @@ -18,6 +18,7 @@ pub(crate) fn build_sysroot( rustup_toolchain_name: Option<&str>, target_triple: String, panic_unwind_support: bool, + keep_sysroot: bool, ) -> Compiler { let _guard = LogGroup::guard("Build sysroot"); @@ -83,6 +84,7 @@ pub(crate) fn build_sysroot( &cg_clif_dylib_path, sysroot_kind, panic_unwind_support, + keep_sysroot, ); host.install_into_sysroot(dist_dir); @@ -98,6 +100,7 @@ pub(crate) fn build_sysroot( &cg_clif_dylib_path, sysroot_kind, panic_unwind_support, + keep_sysroot, ) .install_into_sysroot(dist_dir); } @@ -148,13 +151,18 @@ fn build_sysroot_for_triple( cg_clif_dylib_path: &CodegenBackend, sysroot_kind: SysrootKind, panic_unwind_support: bool, + keep_sysroot: bool, ) -> SysrootTarget { match sysroot_kind { SysrootKind::None => SysrootTarget { triple: compiler.triple, libs: vec![] }, SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler), - SysrootKind::Clif => { - build_clif_sysroot_for_triple(dirs, compiler, cg_clif_dylib_path, panic_unwind_support) - } + SysrootKind::Clif => build_clif_sysroot_for_triple( + dirs, + compiler, + cg_clif_dylib_path, + panic_unwind_support, + keep_sysroot, + ), } } @@ -195,12 +203,13 @@ fn build_clif_sysroot_for_triple( mut compiler: Compiler, cg_clif_dylib_path: &CodegenBackend, panic_unwind_support: bool, + keep_sysroot: bool, ) -> SysrootTarget { let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join("release"); - if !config::get_bool("keep_sysroot") { + if !keep_sysroot { let sysroot_src_orig = get_default_sysroot(&compiler.rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); diff --git a/build_system/main.rs b/build_system/main.rs index 852fda950d88b..04f4a0bde9167 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -79,6 +79,7 @@ fn main() { let mut out_dir = std::env::current_dir().unwrap(); let mut download_dir = None; let mut sysroot_kind = SysrootKind::Clif; + let mut keep_sysroot = false; let mut use_unstable_features = true; let mut panic_unwind_support = false; let mut frozen = false; @@ -105,6 +106,7 @@ fn main() { None => arg_error!("--sysroot requires argument"), } } + "--keep-sysroot" => keep_sysroot = true, "--no-unstable-features" => use_unstable_features = false, "--panic-unwind-support" => panic_unwind_support = true, "--frozen" => frozen = true, @@ -217,6 +219,7 @@ fn main() { sysroot_kind, use_unstable_features, panic_unwind_support, + keep_sysroot, &skip_tests.iter().map(|test| &**test).collect::>(), &cg_clif_dylib, &bootstrap_host_compiler, @@ -236,6 +239,7 @@ fn main() { rustup_toolchain_name.as_deref(), &bootstrap_host_compiler, panic_unwind_support, + keep_sysroot, ); } Command::Build => { @@ -247,6 +251,7 @@ fn main() { rustup_toolchain_name.as_deref(), target_triple, panic_unwind_support, + keep_sysroot, ); } Command::Bench => { @@ -258,6 +263,7 @@ fn main() { rustup_toolchain_name.as_deref(), target_triple, panic_unwind_support, + keep_sysroot, ); bench::benchmark(&dirs, &compiler); } diff --git a/build_system/tests.rs b/build_system/tests.rs index bc7c8d1d5d1f3..2c71d835d4e68 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -269,6 +269,7 @@ pub(crate) fn run_tests( sysroot_kind: SysrootKind, use_unstable_features: bool, panic_unwind_support: bool, + keep_sysroot: bool, skip_tests: &[&str], cg_clif_dylib: &CodegenBackend, bootstrap_host_compiler: &Compiler, @@ -288,6 +289,7 @@ pub(crate) fn run_tests( rustup_toolchain_name, target_triple.clone(), panic_unwind_support, + keep_sysroot, ); let runner = TestRunner::new( @@ -322,6 +324,7 @@ pub(crate) fn run_tests( rustup_toolchain_name, target_triple.clone(), panic_unwind_support, + keep_sysroot, ); let mut runner = TestRunner::new( diff --git a/build_system/usage.txt b/build_system/usage.txt index 572fe78058101..62fe14e1d2479 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -15,6 +15,11 @@ OPTIONS: `clif` will build the standard library using Cranelift. `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM. + --keep-sysroot + Disable cleaning of the sysroot dir to cause old compiled artifacts to be re-used when + the sysroot source hasn't changed. This is useful when the codegen backend hasn't been + modified. + --out-dir DIR Specify the directory in which the download, build and dist directories are stored. By default this is the working directory. diff --git a/config.txt b/config.txt index 7c516e2164b40..cffd850b794f5 100644 --- a/config.txt +++ b/config.txt @@ -1,12 +1,5 @@ # This file allows configuring the build system. -# Disables cleaning of the sysroot dir. This will cause old compiled artifacts to be re-used when -# the sysroot source hasn't changed. This is useful when the codegen backend hasn't been modified. -# This option can be changed while the build system is already running for as long as sysroot -# building hasn't started yet. -#keep_sysroot - - # Testsuite # # Each test suite item has a corresponding key here. The default is to run all tests. From 1629cba0c275512e09a0d7bd1f8588f1f470d94f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:07:50 +0200 Subject: [PATCH 25/51] Introduce SysrootConfig struct --- build_system/abi_cafe.rs | 11 +++---- build_system/build_sysroot.rs | 54 +++++++++++++++++------------------ build_system/main.rs | 42 ++++++++++----------------- build_system/tests.rs | 19 +++++------- 4 files changed, 53 insertions(+), 73 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index 8db09f08063a8..eaa6790f5f6b9 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -1,7 +1,8 @@ +use crate::build_sysroot::SysrootConfig; use crate::path::Dirs; use crate::prepare::GitRepo; use crate::utils::{CargoProject, Compiler, spawn_and_wait}; -use crate::{CodegenBackend, SysrootKind, build_sysroot}; +use crate::{CodegenBackend, build_sysroot}; static ABI_CAFE_REPO: GitRepo = GitRepo::github( "Gankra", @@ -15,13 +16,11 @@ static ABI_CAFE_REPO: GitRepo = GitRepo::github( static ABI_CAFE: CargoProject = CargoProject::new(ABI_CAFE_REPO.source_dir(), "abi_cafe_target"); pub(crate) fn run( - sysroot_kind: SysrootKind, + sysroot_config: &SysrootConfig, dirs: &Dirs, cg_clif_dylib: &CodegenBackend, rustup_toolchain_name: Option<&str>, bootstrap_host_compiler: &Compiler, - panic_unwind_support: bool, - keep_sysroot: bool, ) { std::fs::create_dir_all(&dirs.download_dir).unwrap(); ABI_CAFE_REPO.fetch(dirs); @@ -30,13 +29,11 @@ pub(crate) fn run( eprintln!("Building sysroot for abi-cafe"); build_sysroot::build_sysroot( dirs, - sysroot_kind, + sysroot_config, cg_clif_dylib, bootstrap_host_compiler, rustup_toolchain_name, bootstrap_host_compiler.triple.clone(), - panic_unwind_support, - keep_sysroot, ); eprintln!("Running abi-cafe"); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 0cee2a975d896..c3886c1d10464 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -2,27 +2,38 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, fs}; +use crate::CodegenBackend; use crate::path::{Dirs, RelPath}; use crate::prepare::apply_patches; use crate::rustc_info::{get_default_sysroot, get_file_name}; use crate::utils::{ CargoProject, Compiler, LogGroup, ensure_empty_dir, spawn_and_wait, try_hard_link, }; -use crate::{CodegenBackend, SysrootKind}; + +pub(crate) struct SysrootConfig { + pub(crate) sysroot_kind: SysrootKind, + pub(crate) panic_unwind_support: bool, + pub(crate) keep_sysroot: bool, +} + +#[derive(Copy, Clone, Debug)] +pub(crate) enum SysrootKind { + None, + Clif, + Llvm, +} pub(crate) fn build_sysroot( dirs: &Dirs, - sysroot_kind: SysrootKind, + config: &SysrootConfig, cg_clif_dylib_src: &CodegenBackend, bootstrap_host_compiler: &Compiler, rustup_toolchain_name: Option<&str>, target_triple: String, - panic_unwind_support: bool, - keep_sysroot: bool, ) -> Compiler { let _guard = LogGroup::guard("Build sysroot"); - eprintln!("[BUILD] sysroot {:?}", sysroot_kind); + eprintln!("[BUILD] sysroot {:?}", config.sysroot_kind); let dist_dir = &dirs.dist_dir; @@ -55,7 +66,7 @@ pub(crate) fn build_sysroot( .arg(&wrapper_path) .arg("-Cstrip=debuginfo") .arg("--check-cfg=cfg(support_panic_unwind)"); - if panic_unwind_support { + if config.panic_unwind_support { build_cargo_wrapper_cmd.arg("--cfg").arg("support_panic_unwind"); } if let Some(rustup_toolchain_name) = &rustup_toolchain_name { @@ -82,9 +93,7 @@ pub(crate) fn build_sysroot( dirs, bootstrap_host_compiler.clone(), &cg_clif_dylib_path, - sysroot_kind, - panic_unwind_support, - keep_sysroot, + config, ); host.install_into_sysroot(dist_dir); @@ -98,9 +107,7 @@ pub(crate) fn build_sysroot( bootstrap_target_compiler }, &cg_clif_dylib_path, - sysroot_kind, - panic_unwind_support, - keep_sysroot, + config, ) .install_into_sysroot(dist_dir); } @@ -149,20 +156,14 @@ fn build_sysroot_for_triple( dirs: &Dirs, compiler: Compiler, cg_clif_dylib_path: &CodegenBackend, - sysroot_kind: SysrootKind, - panic_unwind_support: bool, - keep_sysroot: bool, + config: &SysrootConfig, ) -> SysrootTarget { - match sysroot_kind { + match config.sysroot_kind { SysrootKind::None => SysrootTarget { triple: compiler.triple, libs: vec![] }, SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler), - SysrootKind::Clif => build_clif_sysroot_for_triple( - dirs, - compiler, - cg_clif_dylib_path, - panic_unwind_support, - keep_sysroot, - ), + SysrootKind::Clif => { + build_clif_sysroot_for_triple(dirs, compiler, cg_clif_dylib_path, config) + } } } @@ -202,14 +203,13 @@ fn build_clif_sysroot_for_triple( dirs: &Dirs, mut compiler: Compiler, cg_clif_dylib_path: &CodegenBackend, - panic_unwind_support: bool, - keep_sysroot: bool, + config: &SysrootConfig, ) -> SysrootTarget { let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join("release"); - if !keep_sysroot { + if !config.keep_sysroot { let sysroot_src_orig = get_default_sysroot(&compiler.rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); @@ -222,7 +222,7 @@ fn build_clif_sysroot_for_triple( // Build sysroot let mut rustflags = vec!["-Zforce-unstable-if-unmarked".to_owned()]; - if !panic_unwind_support { + if !config.panic_unwind_support { rustflags.push("-Cpanic=abort".to_owned()); } match cg_clif_dylib_path { diff --git a/build_system/main.rs b/build_system/main.rs index 04f4a0bde9167..aa3163cdc7cbe 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -5,7 +5,8 @@ use std::path::PathBuf; use std::{env, process}; -use self::utils::Compiler; +use crate::build_sysroot::{SysrootConfig, SysrootKind}; +use crate::utils::Compiler; mod abi_cafe; mod bench; @@ -42,13 +43,6 @@ enum Command { CheckTodo, } -#[derive(Copy, Clone, Debug)] -enum SysrootKind { - None, - Clif, - Llvm, -} - #[derive(Clone, Debug)] enum CodegenBackend { Local(PathBuf), @@ -78,10 +72,12 @@ fn main() { let mut out_dir = std::env::current_dir().unwrap(); let mut download_dir = None; - let mut sysroot_kind = SysrootKind::Clif; - let mut keep_sysroot = false; + let mut sysroot_config = SysrootConfig { + sysroot_kind: SysrootKind::Clif, + panic_unwind_support: false, + keep_sysroot: false, + }; let mut use_unstable_features = true; - let mut panic_unwind_support = false; let mut frozen = false; let mut skip_tests = vec![]; let mut use_backend = None; @@ -98,7 +94,7 @@ fn main() { }))); } "--sysroot" => { - sysroot_kind = match args.next().as_deref() { + sysroot_config.sysroot_kind = match args.next().as_deref() { Some("none") => SysrootKind::None, Some("clif") => SysrootKind::Clif, Some("llvm") => SysrootKind::Llvm, @@ -106,9 +102,9 @@ fn main() { None => arg_error!("--sysroot requires argument"), } } - "--keep-sysroot" => keep_sysroot = true, + "--keep-sysroot" => sysroot_config.keep_sysroot = true, "--no-unstable-features" => use_unstable_features = false, - "--panic-unwind-support" => panic_unwind_support = true, + "--panic-unwind-support" => sysroot_config.panic_unwind_support = true, "--frozen" => frozen = true, "--skip-test" => { // FIXME check that all passed in tests actually exist @@ -206,7 +202,7 @@ fn main() { &dirs, &bootstrap_host_compiler, use_unstable_features, - panic_unwind_support, + sysroot_config.panic_unwind_support, )) }; match command { @@ -216,10 +212,8 @@ fn main() { Command::Test => { tests::run_tests( &dirs, - sysroot_kind, + &sysroot_config, use_unstable_features, - panic_unwind_support, - keep_sysroot, &skip_tests.iter().map(|test| &**test).collect::>(), &cg_clif_dylib, &bootstrap_host_compiler, @@ -233,37 +227,31 @@ fn main() { process::exit(1); } abi_cafe::run( - sysroot_kind, + &sysroot_config, &dirs, &cg_clif_dylib, rustup_toolchain_name.as_deref(), &bootstrap_host_compiler, - panic_unwind_support, - keep_sysroot, ); } Command::Build => { build_sysroot::build_sysroot( &dirs, - sysroot_kind, + &sysroot_config, &cg_clif_dylib, &bootstrap_host_compiler, rustup_toolchain_name.as_deref(), target_triple, - panic_unwind_support, - keep_sysroot, ); } Command::Bench => { let compiler = build_sysroot::build_sysroot( &dirs, - sysroot_kind, + &sysroot_config, &cg_clif_dylib, &bootstrap_host_compiler, rustup_toolchain_name.as_deref(), target_triple, - panic_unwind_support, - keep_sysroot, ); bench::benchmark(&dirs, &compiler); } diff --git a/build_system/tests.rs b/build_system/tests.rs index 2c71d835d4e68..adaae8b24115c 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -2,12 +2,13 @@ use std::ffi::OsStr; use std::path::PathBuf; use std::process::Command; +use crate::build_sysroot::SysrootConfig; use crate::path::{Dirs, RelPath}; use crate::prepare::{GitRepo, apply_patches}; use crate::rustc_info::get_default_sysroot; use crate::shared_utils::rustflags_from_env; use crate::utils::{CargoProject, Compiler, LogGroup, ensure_empty_dir, spawn_and_wait}; -use crate::{CodegenBackend, SysrootKind, build_sysroot, config}; +use crate::{CodegenBackend, build_sysroot, config}; static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::build("example"); @@ -266,10 +267,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ pub(crate) fn run_tests( dirs: &Dirs, - sysroot_kind: SysrootKind, + sysroot_config: &SysrootConfig, use_unstable_features: bool, - panic_unwind_support: bool, - keep_sysroot: bool, skip_tests: &[&str], cg_clif_dylib: &CodegenBackend, bootstrap_host_compiler: &Compiler, @@ -283,20 +282,18 @@ pub(crate) fn run_tests( if config::get_bool("testsuite.no_sysroot") && !skip_tests.contains(&"testsuite.no_sysroot") { let target_compiler = build_sysroot::build_sysroot( dirs, - SysrootKind::None, + sysroot_config, cg_clif_dylib, bootstrap_host_compiler, rustup_toolchain_name, target_triple.clone(), - panic_unwind_support, - keep_sysroot, ); let runner = TestRunner::new( dirs.clone(), target_compiler, use_unstable_features, - panic_unwind_support, + sysroot_config.panic_unwind_support, skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source.clone(), @@ -318,20 +315,18 @@ pub(crate) fn run_tests( if run_base_sysroot || run_extended_sysroot { let target_compiler = build_sysroot::build_sysroot( dirs, - sysroot_kind, + sysroot_config, cg_clif_dylib, bootstrap_host_compiler, rustup_toolchain_name, target_triple.clone(), - panic_unwind_support, - keep_sysroot, ); let mut runner = TestRunner::new( dirs.clone(), target_compiler, use_unstable_features, - panic_unwind_support, + sysroot_config.panic_unwind_support, skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source, From 49795a11366eee7b6361ad74939a52e0bb33c1c2 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:31:11 +0200 Subject: [PATCH 26/51] Rustup to rustc 1.98.0-nightly (4c9d2bfe4 2026-07-01) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index ba32d5fd58d5e..5ebc028d6c4f3 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-06-29" +channel = "nightly-2026-07-02" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From 6c83401cee6c3d3e7583c9938ed66dd00fbdc07f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:24:50 +0200 Subject: [PATCH 27/51] Update list of ignored rustc tests --- scripts/test_rustc_tests.sh | 42 ++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 09046cdf8c135..0644dd6d4c5b7 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -26,16 +26,15 @@ git checkout -- tests/ui/entry-point/auxiliary/bad_main_functions.rs # ================ # vendor intrinsics -rm tests/ui/asm/x86_64/evex512-implicit-feature.rs # unimplemented AVX512 x86 vendor intrinsic rm tests/ui/simd/dont-invalid-bitcast-x86_64.rs # unimplemented llvm.x86.sse41.round.ps rm tests/ui/simd/intrinsic/generic-arithmetic-pass.rs # unimplemented simd_funnel_{shl,shr} rm -r tests/ui/scalable-vectors # scalable vectors are unsupported # exotic linkages -rm -r tests/ui/linkage* rm tests/incremental/hashes/function_interfaces.rs rm tests/incremental/hashes/statics.rs rm -r tests/run-make/naked-symbol-visibility +rm tests/ui/linkage-attr/linkage-attr-does-not-panic-llvm-issue-33992.rs # variadic arguments rm tests/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs @@ -53,13 +52,13 @@ rm tests/ui/delegation/fn-header.rs rm tests/ui/c-variadic/roundtrip.rs # inline assembly features -rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly rm tests/ui/asm/global-asm-mono-sym-fn.rs # same rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported rm tests/ui/asm/label-operand.rs # same rm tests/ui/asm/may_unwind.rs # asm unwinding not supported rm tests/ui/asm/aarch64/may_unwind.rs # same +rm tests/ui/asm/x86_64/may_unwind.rs # same # misc unimplemented things rm tests/ui/target-feature/missing-plusminus.rs # error not implemented @@ -68,8 +67,6 @@ rm -r tests/run-make/split-debuginfo # same rm -r tests/run-make/target-specs # i686 not supported by Cranelift rm -r tests/run-make/mismatching-target-triples # same rm tests/ui/simd/simd-bitmask-notpow2.rs # non-pow-of-2 simd vector sizes -rm -r tests/run-make/used-proc-macro # used(linker) isn't supported yet -rm tests/ui/linking/no-gc-encapsulation-symbols.rs # same rm tests/ui/attributes/fn-align-dyn.rs # per-function alignment not supported rm -r tests/ui/explicit-tail-calls # tail calls rm -r tests/run-make/pointer-auth-link-with-c # pointer auth @@ -105,8 +102,7 @@ rm tests/ui/linking/executable-no-mangle-strip.rs # requires --gc-sections to wo # backend specific tests # ====================== -rm tests/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM -rm -r tests/run-make/cross-lang-lto # same +rm -r tests/run-make/cross-lang-lto # requires LLVM rm -r tests/run-make/volatile-intrinsics # same rm -r tests/run-make/llvm-ident # same rm -r tests/run-make/no-builtins-attribute # same @@ -115,8 +111,10 @@ rm -r tests/run-make/llvm-location-discriminator-limit-dummy-span # same rm tests/ui/abi/stack-protector.rs # requires stack protector support rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific -rm -r tests/ui/codegen/remark-flag-functionality.rs # same +rm tests/ui/codegen/remark-flag-functionality.rs # same rm -r tests/run-make/print-to-output # requires --print relocation-models +rm tests/ui/abi/rust-preserve-none-cc.rs # extern "rust-preserve-none" is LLVM specific +rm tests/ui/abi/rust-tail-cc.rs # extern "rust-tail" is LLVM specific # requires asm, llvm-ir and/or llvm-bc emit support # ============================================= @@ -135,19 +133,16 @@ rm -r tests/run-make/artifact-incr-cache-no-obj rm -r tests/run-make/emit rm -r tests/run-make/llvm-outputs rm -r tests/run-make/panic-impl-transitive -rm -r tests/ui/debuginfo/debuginfo-emit-llvm-ir-and-split-debuginfo.rs -rm -r tests/ui/statics/issue-91050-1.rs -rm -r tests/ui/statics/issue-91050-2.rs +rm tests/ui/debuginfo/debuginfo-emit-llvm-ir-and-split-debuginfo.rs +rm tests/ui/statics/issue-91050-1.rs +rm tests/ui/statics/issue-91050-2.rs # giving different but possibly correct results # ============================================= -rm tests/ui/mir/mir_misc_casts.rs # depends on deduplication of constants rm tests/ui/mir/mir_raw_fat_ptr.rs # same -rm tests/ui/consts/issue-33537.rs # same rm tests/ui/consts/const-mut-refs-crate.rs # same rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift rm -r tests/run-make/short-ice # ICE backtrace begin/end marker mismatch -rm -r tests/run-make/naked-dead-code-elimination # function not eliminated rm tests/ui/codegen/huge-stacks.rs # Cranelift doesn't allow stack frames to exceed 4GB rm -r tests/run-make/doctests-test_harness # different thread names likely caused by -Zpanic-abort-tests @@ -165,26 +160,25 @@ rm -r tests/run-make/print-request-help-stable-unstable # same rm -r tests/run-make/issue-149402-suggest-unresolve # same rm -r tests/run-make/incr-add-rust-src-component rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path -rm -r tests/run-make/export/extern-opt # something about rustc version mismatches -rm -r tests/run-make/export # same -rm -r tests/ui/compiletest-self-test/compile-flags-incremental.rs # needs compiletest compiled with panic=unwind -rm -r tests/ui/extern/extern-types-field-offset.rs # expects /rustc/ rather than /rustc/FAKE_PREFIX -rm -r tests/ui/process/println-with-broken-pipe.rs # same -rm tests/codegen-units/item-collection/opaque-return-impls.rs # extra mono item. possibly due to other configuration +rm -r tests/run-make/export # something about rustc version mismatches +rm tests/ui/compiletest-self-test/compile-flags-incremental.rs # needs compiletest compiled with panic=unwind +rm tests/ui/extern/extern-types-field-offset.rs # expects /rustc/ rather than /rustc/FAKE_PREFIX +rm tests/ui/process/println-with-broken-pipe.rs # same # genuine bugs # ============ rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort -rm -r tests/run-make/staticlib-hide-internal-symbols # -Zstaticlib-hide-internal-symbols seems to be broken -rm -r tests/run-make/staticlib-rename-internal-symbols # needs files from staticlib-hide-internal-symbols # bugs in the test suite # ====================== rm tests/ui/process/nofile-limit.rs # FIXME some AArch64 linking issue rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables -rm -r tests/incremental/extern_static/issue-49153.rs # assumes reference to undefined static gets optimized away -rm tests/ui/std/linux-getrandom-fallback.rs # missing needs-unwind +rm tests/incremental/extern_static/issue-49153.rs # assumes reference to undefined static gets optimized away +rm tests/ui/linkage-attr/raw-dylib/elf/glibc-x86_64.rs # implicitly depends on libcore getting optimized away to avoid symbol refs +rm tests/ui/thread-local/thread-local-issue-37508.rs # incorrect rust_eh_personality signature +rm -r tests/run-make/staticlib-hide-internal-symbols # missing needs-unwind +rm -r tests/run-make/staticlib-rename-internal-symbols # needs files from staticlib-hide-internal-symbols rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # really slow with unoptimized libstd rm tests/ui/process/process-panic-after-fork.rs # same From 4b57ec974ffe7035f975a8b7a64147c9a05fbff2 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:19:57 +0000 Subject: [PATCH 28/51] Fix rustc test suite on arm64 --- scripts/test_rustc_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 0644dd6d4c5b7..d1ca140664b61 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -169,6 +169,7 @@ rm tests/ui/process/println-with-broken-pipe.rs # same # ============ rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort +rm -r tests/run-make/used-proc-macro # doesn't work on arm64 for some reason # bugs in the test suite # ====================== From fca1bc4edd8cc5290230f43473f710594b30bee5 Mon Sep 17 00:00:00 2001 From: Raushan kumar Date: Sun, 5 Jul 2026 07:00:04 +0000 Subject: [PATCH 29/51] Distinguish null reference production from null pointer dereference Introduce a dedicated panic for null reference production to distinguish producing a reference from a null pointer from an actual null pointer dereference. This emits a dedicated panic message for null reference production while preserving the existing panic for actual dereferences. --- src/base.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/base.rs b/src/base.rs index 467eceea221c8..b37fb213d945a 100644 --- a/src/base.rs +++ b/src/base.rs @@ -421,6 +421,17 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { source_info.span, ) } + AssertKind::NullReferenceConstructed => { + let location = fx.get_caller_location(source_info).load_scalar(fx); + + codegen_panic_inner( + fx, + rustc_hir::LangItem::PanicNullReferenceConstructed, + &[location], + *unwind, + source_info.span, + ) + } AssertKind::InvalidEnumConstruction(source) => { let source = codegen_operand(fx, source).load_scalar(fx); let location = fx.get_caller_location(source_info).load_scalar(fx); From 23c7935bb1c9627d5946400e1b6888bfc9d64ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Soares?= <37777652+Dnreikronos@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:46:17 -0300 Subject: [PATCH 30/51] codegen_cranelift: Fix backend limit diagnostics (#1673) --- src/base.rs | 14 +++++--------- src/driver/aot.rs | 11 ++++++----- src/driver/jit.rs | 1 + 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/base.rs b/src/base.rs index 6c6122501c4a0..22695a517b7fc 100644 --- a/src/base.rs +++ b/src/base.rs @@ -7,6 +7,7 @@ use cranelift_module::ModuleError; use rustc_ast::InlineAsmOptions; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_errors::DiagCtxtHandle; use rustc_index::IndexVec; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::adjustment::PointerCoercion; @@ -148,6 +149,7 @@ pub(crate) fn codegen_fn<'tcx>( pub(crate) fn compile_fn( prof: &SelfProfilerRef, + dcx: DiagCtxtHandle<'_>, output_filenames: &OutputFilenames, should_write_ir: bool, cached_context: &mut Context, @@ -192,25 +194,19 @@ pub(crate) fn compile_fn( match module.define_function(codegened_func.func_id, context) { Ok(()) => {} Err(ModuleError::Compilation(CodegenError::ImplLimitExceeded)) => { - let early_dcx = rustc_session::EarlyDiagCtxt::new( - rustc_session::config::ErrorOutputType::default(), - ); - early_dcx.early_fatal(format!( + dcx.fatal(format!( "backend implementation limit exceeded while compiling {name}", name = codegened_func.symbol_name )); } Err(ModuleError::Compilation(CodegenError::Verifier(err))) => { - let early_dcx = rustc_session::EarlyDiagCtxt::new( - rustc_session::config::ErrorOutputType::default(), - ); - let _ = early_dcx.early_err(format!("{:?}", err)); + let raw_error = format!("{:?}", err); let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( &context.func, Some(Box::new(&clif_comments)), err, ); - early_dcx.early_fatal(format!("cranelift verify error:\n{}", pretty_error)); + dcx.fatal(format!("cranelift verify error:\n{raw_error}\n{pretty_error}")); } Err(err) => { let mut clif = format_clif_ir_header(module.isa(), &codegened_func.symbol_name); diff --git a/src/driver/aot.rs b/src/driver/aot.rs index f4acfff8bb9fe..d6c25cf524a5c 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -17,7 +17,7 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::{ExtraBackendMethods, WriteBackendMethods}; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_errors::DiagCtxt; +use rustc_errors::{DiagCtxt, DiagCtxtHandle}; use rustc_hir::attrs::Linkage as RLinkage; use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -205,6 +205,7 @@ fn codegen_cgu(tcx: TyCtxt<'_>, cgu_name: Symbol) -> AotModule { fn compile_cgu( prof: &SelfProfilerRef, + dcx: DiagCtxtHandle<'_>, output_filenames: &OutputFilenames, should_write_ir: bool, mut aot_module: AotModule, @@ -220,6 +221,7 @@ fn compile_cgu( for codegened_func in aot_module.codegened_functions { crate::base::compile_fn( &prof, + dcx, &output_filenames, should_write_ir, &mut cached_context, @@ -370,18 +372,17 @@ impl WriteBackendMethods for AotDriver { module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule { + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); compile_cgu( prof, + dcx.handle(), &cgcx.output_filenames, config.emit_ir, module.module_llvm, module.name, module.kind, ) - .unwrap_or_else(|err| { - let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); - dcx.handle().fatal(err) - }) + .unwrap_or_else(|err| dcx.handle().fatal(err)) } fn serialize_module(_module: Self::Module, _is_thin: bool) -> Self::ModuleBuffer { diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 7d6ece02e4a62..32f60615844bc 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -157,6 +157,7 @@ fn codegen_and_compile_fn<'tcx>( let mut global_asm = String::new(); crate::base::compile_fn( &tcx.prof, + tcx.dcx(), output_filenames, should_write_ir, cached_context, From 44273d665a74d9ce901343287cfa2ec1344ac319 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Wed, 1 Jul 2026 08:33:26 -0700 Subject: [PATCH 31/51] Carry the `b_offset` inside `BackendRepr::ScalarPair` --- src/abi/mod.rs | 2 +- src/abi/pass_mode.rs | 4 ++-- src/base.rs | 2 +- src/intrinsics/mod.rs | 8 ++++++-- src/value_and_place.rs | 23 ++++++++++------------- src/vtable.rs | 15 ++++++++------- 6 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 012951098fa15..ec17e72900dfb 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -220,7 +220,7 @@ fn make_local_place<'tcx>( ); } let place = if is_ssa { - if let BackendRepr::ScalarPair(_, _) = layout.backend_repr { + if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = layout.backend_repr { CPlace::new_var_pair(fx, local, layout) } else { CPlace::new_var(fx, local, layout) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 612f89e6a4217..75d7e4d9fd500 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -112,7 +112,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { _ => unreachable!("{:?}", self.layout.backend_repr), }, PassMode::Pair(attrs_a, attrs_b) => match self.layout.backend_repr { - BackendRepr::ScalarPair(a, b) => { + BackendRepr::ScalarPair { a, b, b_offset: _ } => { let a = scalar_to_clif_type(tcx, a); let b = scalar_to_clif_type(tcx, b); smallvec![ @@ -167,7 +167,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { _ => unreachable!("{:?}", self.layout.backend_repr), }, PassMode::Pair(attrs_a, attrs_b) => match self.layout.backend_repr { - BackendRepr::ScalarPair(a, b) => { + BackendRepr::ScalarPair { a, b, b_offset: _ } => { let a = scalar_to_clif_type(tcx, a); let b = scalar_to_clif_type(tcx, b); ( diff --git a/src/base.rs b/src/base.rs index b37fb213d945a..bf6a98866cd12 100644 --- a/src/base.rs +++ b/src/base.rs @@ -695,7 +695,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: } UnOp::PtrMetadata => match layout.backend_repr { BackendRepr::Scalar(_) => CValue::zst(dest_layout), - BackendRepr::ScalarPair(_, _) => { + BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } => { CValue::by_val(operand.load_scalar_pair(fx).1, dest_layout) } _ => bug!("Unexpected `PtrToMetadata` operand: {operand:?}"), diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index e6b8c537d8451..d4bb8bd019331 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -572,7 +572,9 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = fx.layout_of(generic_args.type_at(0)); // Note: Can't use is_unsized here as truly unsized types need to take the fixed size // branch - let meta = if let BackendRepr::ScalarPair(_, _) = ptr.layout().backend_repr { + let meta = if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = + ptr.layout().backend_repr + { Some(ptr.load_scalar_pair(fx).1) } else { None @@ -586,7 +588,9 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = fx.layout_of(generic_args.type_at(0)); // Note: Can't use is_unsized here as truly unsized types need to take the fixed size // branch - let meta = if let BackendRepr::ScalarPair(_, _) = ptr.layout().backend_repr { + let meta = if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = + ptr.layout().backend_repr + { Some(ptr.load_scalar_pair(fx).1) } else { None diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 995a70f24240b..d7248acfafef1 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -55,8 +55,7 @@ fn codegen_field<'tcx>( } } -fn scalar_pair_calculate_b_offset(tcx: TyCtxt<'_>, a_scalar: Scalar, b_scalar: Scalar) -> Offset32 { - let b_offset = a_scalar.size(&tcx).align_to(b_scalar.default_align(&tcx).abi); +fn scalar_pair_convert_b_offset(b_offset: Size) -> Offset32 { Offset32::new(b_offset.bytes().try_into().unwrap()) } @@ -159,11 +158,11 @@ impl<'tcx> CValue<'tcx> { let layout = self.1; match self.0 { CValueInner::ByRef(ptr, None) => { - let (a_scalar, b_scalar) = match layout.backend_repr { - BackendRepr::ScalarPair(a, b) => (a, b), + let (a_scalar, b_scalar, b_offset) = match layout.backend_repr { + BackendRepr::ScalarPair { a, b, b_offset } => (a, b, b_offset), _ => unreachable!("load_scalar_pair({:?})", self), }; - let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let b_offset = scalar_pair_convert_b_offset(b_offset); let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar); let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar); let mut flags = MemFlags::new(); @@ -189,7 +188,7 @@ impl<'tcx> CValue<'tcx> { match self.0 { CValueInner::ByVal(_) => unreachable!(), CValueInner::ByValPair(val1, val2) => match layout.backend_repr { - BackendRepr::ScalarPair(_, _) => { + BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } => { let val = match field.as_u32() { 0 => val1, 1 => val2, @@ -580,7 +579,7 @@ impl<'tcx> CPlace<'tcx> { } CPlaceInner::VarPair(_local, var1, var2) => { let (data1, data2) = match from.1.backend_repr { - BackendRepr::ScalarPair(_, _) => { + BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } => { CValue(from.0, dst_layout).load_scalar_pair(fx) } _ => { @@ -607,9 +606,8 @@ impl<'tcx> CPlace<'tcx> { to_ptr.store(fx, val, flags); } CValueInner::ByValPair(val1, val2) => match from.layout().backend_repr { - BackendRepr::ScalarPair(a_scalar, b_scalar) => { - let b_offset = - scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + BackendRepr::ScalarPair { a: _, b: _, b_offset } => { + let b_offset = scalar_pair_convert_b_offset(b_offset); to_ptr.store(fx, val1, flags); to_ptr.offset(fx, b_offset).store(fx, val2, flags); } @@ -627,9 +625,8 @@ impl<'tcx> CPlace<'tcx> { to_ptr.store(fx, val, flags); return; } - BackendRepr::ScalarPair(a_scalar, b_scalar) => { - let b_offset = - scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + BackendRepr::ScalarPair { a: _, b: _, b_offset } => { + let b_offset = scalar_pair_convert_b_offset(b_offset); let (val1, val2) = from.load_scalar_pair(fx); to_ptr.store(fx, val1, flags); to_ptr.offset(fx, b_offset).store(fx, val2, flags); diff --git a/src/vtable.rs b/src/vtable.rs index b5d241d8f39f2..3310e4e3a2228 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -56,13 +56,14 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>( } } - let (ptr, vtable) = if let BackendRepr::ScalarPair(_, _) = arg.layout().backend_repr { - let (ptr, vtable) = arg.load_scalar_pair(fx); - (Pointer::new(ptr), vtable) - } else { - let (ptr, vtable) = arg.try_to_ptr().unwrap(); - (ptr, vtable.unwrap()) - }; + let (ptr, vtable) = + if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = arg.layout().backend_repr { + let (ptr, vtable) = arg.load_scalar_pair(fx); + (Pointer::new(ptr), vtable) + } else { + let (ptr, vtable) = arg.try_to_ptr().unwrap(); + (ptr, vtable.unwrap()) + }; let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes(); let func_ref = fx.bcx.ins().load( From 28eb6979d6ce3392434387a9666d532a1e69aada Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:26:44 +0200 Subject: [PATCH 32/51] Revert an error change --- src/base.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/base.rs b/src/base.rs index 22695a517b7fc..4aa7bb3fcb3f4 100644 --- a/src/base.rs +++ b/src/base.rs @@ -200,13 +200,13 @@ pub(crate) fn compile_fn( )); } Err(ModuleError::Compilation(CodegenError::Verifier(err))) => { - let raw_error = format!("{:?}", err); + dcx.err(format!("{err:?}")); let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( &context.func, Some(Box::new(&clif_comments)), err, ); - dcx.fatal(format!("cranelift verify error:\n{raw_error}\n{pretty_error}")); + dcx.fatal(format!("cranelift verify error:\n{pretty_error}")); } Err(err) => { let mut clif = format_clif_ir_header(module.isa(), &codegened_func.symbol_name); From 5de4124bce2c5c86a4cce0ac3a7e97f17113da76 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:39:16 +0200 Subject: [PATCH 33/51] Rustup to rustc 1.99.0-nightly (f10db292a 2026-07-07) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5ebc028d6c4f3..455f08483c2f2 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-07-02" +channel = "nightly-2026-07-08" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From 5bbb9e4f6618aa7c32bde230875fa36608e60bdf Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:47:39 +0200 Subject: [PATCH 34/51] Support -Zforce-intrinsic-fallback --- src/intrinsics/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index f7cfc822293f3..e845b2eca23da 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -276,6 +276,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( target: Option, source_info: mir::SourceInfo, ) -> Result<(), Instance<'tcx>> { + // When `-Zforce-intrinsic-fallback` is enabled, always use the fallback body if it exists, + if fx.tcx.sess.opts.unstable_opts.force_intrinsic_fallback + && let Some(def) = fx.tcx.intrinsic(instance.def_id()) + && !def.must_be_overridden + { + return Err(Instance::new_raw(instance.def_id(), instance.args)); + } + let intrinsic = fx.tcx.item_name(instance.def_id()); let instance_args = instance.args; From 952e3ff6f25a3b02d6fab4a960aa8208257f2f62 Mon Sep 17 00:00:00 2001 From: "addie.sh" Date: Wed, 17 Jun 2026 21:29:10 +0200 Subject: [PATCH 35/51] place FnDef behind a binder, instantiate w/ Dummy --- src/abi/mod.rs | 2 +- src/base.rs | 2 +- src/inline_asm.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index ec17e72900dfb..4871e80a75645 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -421,7 +421,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( fx.tcx, ty::TypingEnv::fully_monomorphized(), def_id, - fn_args, + fn_args.no_bound_vars().unwrap(), source_info.span, ); diff --git a/src/base.rs b/src/base.rs index bf6a98866cd12..9bf379c03687c 100644 --- a/src/base.rs +++ b/src/base.rs @@ -717,7 +717,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: fx.tcx, ty::TypingEnv::fully_monomorphized(), def_id, - args, + args.no_bound_vars().unwrap(), ) .unwrap(), ); diff --git a/src/inline_asm.rs b/src/inline_asm.rs index f9fc7002be87b..cc849759754ee 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -117,7 +117,7 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>( fx.tcx, ty::TypingEnv::fully_monomorphized(), def_id, - args, + args.no_bound_vars().unwrap(), ) .unwrap(); let symbol = fx.tcx.symbol_name(instance); From 720e6834d620d4ee213881280da88e7beab0714a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:14:28 +0200 Subject: [PATCH 36/51] Rustup to rustc 1.99.0-nightly (375b1431b 2026-07-10) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 455f08483c2f2..84d3f1a99e086 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-07-08" +channel = "nightly-2026-07-11" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From 1302a4fa435599d34a291aebf125e66748b079e5 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:23:16 +0200 Subject: [PATCH 37/51] Minor cleanup --- src/abi/mod.rs | 2 +- src/base.rs | 2 +- src/intrinsics/mod.rs | 8 ++------ src/value_and_place.rs | 14 +++++--------- src/vtable.rs | 15 +++++++-------- 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 18cb5cc6caec0..e4ae8c27309fe 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -220,7 +220,7 @@ fn make_local_place<'tcx>( ); } let place = if is_ssa { - if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = layout.backend_repr { + if let BackendRepr::ScalarPair { .. } = layout.backend_repr { CPlace::new_var_pair(fx, local, layout) } else { CPlace::new_var(fx, local, layout) diff --git a/src/base.rs b/src/base.rs index 0a1f41a416e32..cd496527c2824 100644 --- a/src/base.rs +++ b/src/base.rs @@ -695,7 +695,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: } UnOp::PtrMetadata => match layout.backend_repr { BackendRepr::Scalar(_) => CValue::zst(dest_layout), - BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } => { + BackendRepr::ScalarPair { .. } => { CValue::by_val(operand.load_scalar_pair(fx).1, dest_layout) } _ => bug!("Unexpected `PtrToMetadata` operand: {operand:?}"), diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index c60cd7bdacf76..c9a2b4ad9cd5c 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -580,9 +580,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = fx.layout_of(generic_args.type_at(0)); // Note: Can't use is_unsized here as truly unsized types need to take the fixed size // branch - let meta = if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = - ptr.layout().backend_repr - { + let meta = if let BackendRepr::ScalarPair { .. } = ptr.layout().backend_repr { Some(ptr.load_scalar_pair(fx).1) } else { None @@ -596,9 +594,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = fx.layout_of(generic_args.type_at(0)); // Note: Can't use is_unsized here as truly unsized types need to take the fixed size // branch - let meta = if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = - ptr.layout().backend_repr - { + let meta = if let BackendRepr::ScalarPair { .. } = ptr.layout().backend_repr { Some(ptr.load_scalar_pair(fx).1) } else { None diff --git a/src/value_and_place.rs b/src/value_and_place.rs index f900966511a86..ae4bcad5c8ebb 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -55,10 +55,6 @@ fn codegen_field<'tcx>( } } -fn scalar_pair_convert_b_offset(b_offset: Size) -> Offset32 { - Offset32::new(b_offset.bytes().try_into().unwrap()) -} - /// A read-only value #[derive(Debug, Copy, Clone)] pub(crate) struct CValue<'tcx>(CValueInner, TyAndLayout<'tcx>); @@ -161,7 +157,7 @@ impl<'tcx> CValue<'tcx> { BackendRepr::ScalarPair { a, b, b_offset } => (a, b, b_offset), _ => unreachable!("load_scalar_pair({:?})", self), }; - let b_offset = scalar_pair_convert_b_offset(b_offset); + let b_offset = Offset32::new(b_offset.bytes().try_into().unwrap()); let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar); let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar); let flags = MemFlagsData::new().with_notrap(); @@ -186,7 +182,7 @@ impl<'tcx> CValue<'tcx> { match self.0 { CValueInner::ByVal(_) => unreachable!(), CValueInner::ByValPair(val1, val2) => match layout.backend_repr { - BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } => { + BackendRepr::ScalarPair { .. } => { let val = match field.as_u32() { 0 => val1, 1 => val2, @@ -577,7 +573,7 @@ impl<'tcx> CPlace<'tcx> { } CPlaceInner::VarPair(_local, var1, var2) => { let (data1, data2) = match from.1.backend_repr { - BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } => { + BackendRepr::ScalarPair { .. } => { CValue(from.0, dst_layout).load_scalar_pair(fx) } _ => { @@ -604,7 +600,7 @@ impl<'tcx> CPlace<'tcx> { } CValueInner::ByValPair(val1, val2) => match from.layout().backend_repr { BackendRepr::ScalarPair { a: _, b: _, b_offset } => { - let b_offset = scalar_pair_convert_b_offset(b_offset); + let b_offset = Offset32::new(b_offset.bytes().try_into().unwrap()); to_ptr.store(fx, val1, flags); to_ptr.offset(fx, b_offset).store(fx, val2, flags); } @@ -623,7 +619,7 @@ impl<'tcx> CPlace<'tcx> { return; } BackendRepr::ScalarPair { a: _, b: _, b_offset } => { - let b_offset = scalar_pair_convert_b_offset(b_offset); + let b_offset = Offset32::new(b_offset.bytes().try_into().unwrap()); let (val1, val2) = from.load_scalar_pair(fx); to_ptr.store(fx, val1, flags); to_ptr.offset(fx, b_offset).store(fx, val2, flags); diff --git a/src/vtable.rs b/src/vtable.rs index 76ea77b4c795a..2bb1944cc246b 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -56,14 +56,13 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>( } } - let (ptr, vtable) = - if let BackendRepr::ScalarPair { a: _, b: _, b_offset: _ } = arg.layout().backend_repr { - let (ptr, vtable) = arg.load_scalar_pair(fx); - (Pointer::new(ptr), vtable) - } else { - let (ptr, vtable) = arg.try_to_ptr().unwrap(); - (ptr, vtable.unwrap()) - }; + let (ptr, vtable) = if let BackendRepr::ScalarPair { .. } = arg.layout().backend_repr { + let (ptr, vtable) = arg.load_scalar_pair(fx); + (Pointer::new(ptr), vtable) + } else { + let (ptr, vtable) = arg.try_to_ptr().unwrap(); + (ptr, vtable.unwrap()) + }; let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes(); let func_ref = fx.bcx.ins().load( From 7685cdee6b890025d866819269ea4d96f4400fd1 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:07:17 +0200 Subject: [PATCH 38/51] Fix f128 ABI on s390x --- src/abi/mod.rs | 55 +++++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index e4ae8c27309fe..2b558a7d09531 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -139,41 +139,50 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { mut returns: Vec, args: &[Value], ) -> Cow<'_, [Value]> { + // FIXME any way to reuse the abi adjustment code in rustc_target? + // Pass i128 arguments by-ref on Windows. - let (params, args): (Vec<_>, Cow<'_, [_]>) = if self.tcx.sess.target.is_like_windows { - let (params, args): (Vec<_>, Vec<_>) = params - .into_iter() - .zip(args) - .map(|(param, &arg)| { - if param.value_type == types::I128 { - let arg_ptr = self.create_stack_slot(16, 16); - arg_ptr.store(self, arg, MemFlagsData::trusted()); - (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self)) - } else { - (param, arg) - } - }) - .unzip(); + let (params, args): (Vec<_>, Cow<'_, [_]>) = + if self.tcx.sess.target.is_like_windows || self.tcx.sess.target.arch == Arch::S390x { + let (params, args): (Vec<_>, Vec<_>) = params + .into_iter() + .zip(args) + .map(|(param, &arg)| { + if param.value_type == types::I128 + || (self.tcx.sess.target.arch == Arch::S390x + && param.value_type == types::F128) + { + let arg_ptr = self.create_stack_slot(16, 16); + arg_ptr.store(self, arg, MemFlagsData::trusted()); + (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self)) + } else { + (param, arg) + } + }) + .unzip(); - (params, args.into()) - } else { - (params, args.into()) - }; + (params, args.into()) + } else { + (params, args.into()) + }; - let ret_single_i128 = returns.len() == 1 && returns[0].value_type == types::I128; - if ret_single_i128 && self.tcx.sess.target.is_like_windows { + if self.tcx.sess.target.is_like_windows + && matches!(*returns, [AbiParam { value_type: types::I128, .. }]) + { // Return i128 using the vector ABI on Windows returns[0].value_type = types::I64X2; let ret = self.lib_call_unadjusted(name, params, returns, &args)[0]; Cow::Owned(vec![codegen_bitcast(self, types::I128, ret)]) - } else if ret_single_i128 && self.tcx.sess.target.arch == Arch::S390x { - // Return i128 using a return area pointer on s390x. + } else if self.tcx.sess.target.arch == Arch::S390x + && matches!(*returns, [AbiParam { value_type: types::I128 | types::F128, .. }]) + { + // Return i128 and f128 using a return area pointer on s390x. let mut params = params; let mut args = args.to_vec(); - params.insert(0, AbiParam::new(self.pointer_type)); + params.insert(0, AbiParam::special(self.pointer_type, ArgumentPurpose::StructReturn)); let ret_ptr = self.create_stack_slot(16, 16); args.insert(0, ret_ptr.get_addr(self)); From e997dc76e75b496620b858dd4b85dbfc2434e75e Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sat, 11 Jul 2026 20:37:02 +0200 Subject: [PATCH 39/51] remove f16-math patch We now have fallbacks via f32 for the intrinsics in core --- .../0029-sysroot_tests-disable-f16-math.patch | 582 ------------------ src/intrinsics/mod.rs | 41 +- 2 files changed, 23 insertions(+), 600 deletions(-) delete mode 100644 patches/0029-sysroot_tests-disable-f16-math.patch diff --git a/patches/0029-sysroot_tests-disable-f16-math.patch b/patches/0029-sysroot_tests-disable-f16-math.patch deleted file mode 100644 index 8d21359aa0043..0000000000000 --- a/patches/0029-sysroot_tests-disable-f16-math.patch +++ /dev/null @@ -1,582 +0,0 @@ -From 285d5716fcfa6d43a3516d899b73bc85da322c25 Mon Sep 17 00:00:00 2001 -From: xonx <119700621+xonx4l@users.noreply.github.com> -Date: Sun, 15 Feb 2026 14:06:49 +0000 -Subject: [PATCH] Disable f16 math tests for cranelift - ---- - coretests/tests/num/floats.rs | 26 +++++++++++++------------- - 1 file changed, 13 insertions(+), 13 deletions(-) - -diff --git a/coretests/tests/num/floats.rs b/coretests/tests/num/floats.rs -index 1d7956b..01e4caa 100644 ---- a/coretests/tests/num/floats.rs -+++ b/coretests/tests/num/floats.rs -@@ -444,7 +444,7 @@ pub(crate) use float_test; - float_test! { - name: num, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -463,7 +463,7 @@ float_test! { - name: num_rem, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -476,7 +476,7 @@ float_test! { - float_test! { - name: nan, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -496,7 +496,7 @@ float_test! { - float_test! { - name: infinity, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -514,7 +514,7 @@ float_test! { - float_test! { - name: neg_infinity, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -532,7 +532,7 @@ float_test! { - float_test! { - name: zero, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -550,7 +550,7 @@ float_test! { - float_test! { - name: neg_zero, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -570,7 +570,7 @@ float_test! { - float_test! { - name: one, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -588,7 +588,7 @@ float_test! { - float_test! { - name: is_nan, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -609,7 +609,7 @@ float_test! { - float_test! { - name: is_infinite, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -630,7 +630,7 @@ float_test! { - float_test! { - name: is_finite, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -651,7 +651,7 @@ float_test! { - float_test! { - name: is_normal, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -673,7 +673,7 @@ float_test! { - float_test! { - name: classify, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - }, - test { - let nan: Float = Float::NAN; -@@ -695,7 +695,7 @@ float_test! { - name: min, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -737,7 +737,7 @@ float_test! { - name: max, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -780,7 +780,7 @@ float_test! { - name: minimum, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -812,7 +812,7 @@ float_test! { - name: maximum, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -845,7 +845,7 @@ float_test! { - name: midpoint, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -898,7 +898,7 @@ float_test! { - attrs: { - const: #[cfg(false)], - // Needs powi -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -929,7 +929,7 @@ float_test! { - name: abs, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -948,7 +948,7 @@ float_test! { - name: copysign, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -964,7 +964,7 @@ float_test! { - attrs: { - const: #[cfg(false)], - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -982,7 +982,7 @@ float_test! { - attrs: { - const: #[cfg(false)], - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -998,7 +998,7 @@ float_test! { - name: floor, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1028,7 +1028,7 @@ float_test! { - name: ceil, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1058,7 +1058,7 @@ float_test! { - name: round, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1089,7 +1089,7 @@ float_test! { - name: round_ties_even, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1120,7 +1120,7 @@ float_test! { - name: trunc, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1150,7 +1150,7 @@ float_test! { - name: fract, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1182,7 +1182,7 @@ float_test! { - name: signum, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1200,7 +1200,7 @@ float_test! { - float_test! { - name: is_sign_positive, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1219,7 +1219,7 @@ float_test! { - float_test! { - name: is_sign_negative, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1238,7 +1238,7 @@ float_test! { - float_test! { - name: next_up, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1269,7 +1269,7 @@ float_test! { - float_test! { - name: next_down, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1303,7 +1303,7 @@ float_test! { - attrs: { - const: #[cfg(false)], - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1321,7 +1321,7 @@ float_test! { - name: clamp_min_greater_than_max, - attrs: { - const: #[cfg(false)], -- f16: #[should_panic, cfg(target_has_reliable_f16)], -+ f16: #[should_panic, cfg(false)], - f32: #[should_panic], - f64: #[should_panic], - f128: #[should_panic, cfg(target_has_reliable_f128)], -@@ -1335,7 +1335,7 @@ float_test! { - name: clamp_min_is_nan, - attrs: { - const: #[cfg(false)], -- f16: #[should_panic, cfg(target_has_reliable_f16)], -+ f16: #[should_panic, cfg(false)], - f32: #[should_panic], - f64: #[should_panic], - f128: #[should_panic, cfg(target_has_reliable_f128)], -@@ -1349,7 +1349,7 @@ float_test! { - name: clamp_max_is_nan, - attrs: { - const: #[cfg(false)], -- f16: #[should_panic, cfg(target_has_reliable_f16)], -+ f16: #[should_panic, cfg(false)], - f32: #[should_panic], - f64: #[should_panic], - f128: #[should_panic, cfg(target_has_reliable_f128)], -@@ -1363,7 +1363,7 @@ float_test! { - name: total_cmp, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1469,7 +1469,7 @@ float_test! { - attrs: { - const: #[cfg(false)], - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1526,7 +1526,7 @@ float_test! { - name: recip, - attrs: { - // Miri only uses softfloats here, so that always works -- f16: #[cfg(any(miri, target_has_reliable_f16_math))], -+ f16: #[cfg(false)], - f128: #[cfg(any(miri, target_has_reliable_f128_math))], - }, - test { -@@ -1549,7 +1549,7 @@ float_test! { - name: powi, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1570,7 +1570,7 @@ float_test! { - name: powf, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1593,7 +1593,7 @@ float_test! { - name: exp, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1614,7 +1614,7 @@ float_test! { - name: exp2, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1634,7 +1634,7 @@ float_test! { - name: ln, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1656,7 +1656,7 @@ float_test! { - name: log, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1681,7 +1681,7 @@ float_test! { - name: log2, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1704,7 +1704,7 @@ float_test! { - name: log10, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1728,7 +1728,7 @@ float_test! { - name: asinh, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1764,7 +1764,7 @@ float_test! { - name: acosh, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1795,7 +1795,7 @@ float_test! { - name: atanh, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1821,7 +1821,7 @@ float_test! { - name: gamma, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1856,7 +1856,7 @@ float_test! { - name: ln_gamma, - attrs: { - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { -@@ -1874,7 +1874,7 @@ float_test! { - float_test! { - name: to_degrees, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1895,7 +1895,7 @@ float_test! { - float_test! { - name: to_radians, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1916,7 +1916,7 @@ float_test! { - float_test! { - name: to_algebraic, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1940,7 +1940,7 @@ float_test! { - float_test! { - name: to_bits_conv, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -1967,7 +1967,7 @@ float_test! { - float_test! { - name: mul_add, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - // FIXME(#140515): mingw has an incorrect fma https://sourceforge.net/p/mingw-w64/bugs/848/ - f32: #[cfg_attr(all(target_os = "windows", target_env = "gnu", not(target_abi = "llvm")), ignore)], - f64: #[cfg_attr(all(target_os = "windows", target_env = "gnu", not(target_abi = "llvm")), ignore)], -@@ -1992,7 +1992,7 @@ float_test! { - float_test! { - name: from, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -2049,7 +2049,7 @@ float_test! { - float_test! { - name: max_exact_integer_constant, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -2091,7 +2091,7 @@ float_test! { - float_test! { - name: min_exact_integer_constant, - attrs: { -- f16: #[cfg(target_has_reliable_f16)], -+ f16: #[cfg(false)], - f128: #[cfg(target_has_reliable_f128)], - }, - test { -@@ -2156,7 +2156,7 @@ float_test! { - attrs: { - // FIXME(f16_f128): add math tests when available - const: #[cfg(false)], -- f16: #[cfg(target_has_reliable_f16_math)], -+ f16: #[cfg(false)], // FIXME(rust-lang/rustc_codegen_cranelift#1622) - f128: #[cfg(target_has_reliable_f128_math)], - }, - test { --- -2.50.1 diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index c9a2b4ad9cd5c..292c8004ef434 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -322,75 +322,92 @@ fn codegen_float_intrinsic_call<'tcx>( ret: CPlace<'tcx>, ) -> bool { let (name, arg_count, ty, clif_ty) = match intrinsic { - sym::expf16 => ("expf16", 1, fx.tcx.types.f16, types::F16), + sym::expf16 => return false, // has a fallback via f32 sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32), sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64), sym::expf128 => ("expf128", 1, fx.tcx.types.f128, types::F128), - sym::exp2f16 => ("exp2f16", 1, fx.tcx.types.f16, types::F16), + + sym::exp2f16 => return false, // has a fallback via f32 sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32), sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64), sym::exp2f128 => ("exp2f128", 1, fx.tcx.types.f128, types::F128), + sym::sqrtf16 => ("sqrtf16", 1, fx.tcx.types.f16, types::F16), sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32), sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64), sym::sqrtf128 => ("sqrtf128", 1, fx.tcx.types.f128, types::F128), + sym::powif16 => ("__powisf2", 2, fx.tcx.types.f16, types::F16), // compiler-builtins sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins - sym::powf16 => ("powf16", 2, fx.tcx.types.f16, types::F16), + + sym::powf16 => return false, // has a fallback via f32 sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32), sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64), sym::powf128 => ("powf128", 2, fx.tcx.types.f128, types::F128), - sym::logf16 => ("logf16", 1, fx.tcx.types.f16, types::F16), + + sym::logf16 => return false, // has a fallback via f32 sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32), sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64), sym::logf128 => ("logf128", 1, fx.tcx.types.f128, types::F128), - sym::log2f16 => ("log2f16", 1, fx.tcx.types.f16, types::F16), + + sym::log2f16 => return false, // has a fallback via f32 sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32), sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64), sym::log2f128 => ("log2f128", 1, fx.tcx.types.f128, types::F128), - sym::log10f16 => ("log10f16", 1, fx.tcx.types.f16, types::F16), + + sym::log10f16 => return false, // has a fallback via f32 sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32), sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64), sym::log10f128 => ("log10f128", 1, fx.tcx.types.f128, types::F128), + sym::fmaf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16), sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64), sym::fmaf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128), + // FIXME: calling `fma` from libc without FMA target feature uses expensive sofware emulation sym::fmuladdf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16), // FIXME: use cranelift intrinsic analogous to llvm.fmuladd.f16 sym::fmuladdf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), // FIXME: use cranelift intrinsic analogous to llvm.fmuladd.f32 sym::fmuladdf64 => ("fma", 3, fx.tcx.types.f64, types::F64), // FIXME: use cranelift intrinsic analogous to llvm.fmuladd.f64 sym::fmuladdf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128), // FIXME: use cranelift intrinsic analogous to llvm.fmuladd.f128 + sym::copysignf16 => ("copysignf16", 2, fx.tcx.types.f16, types::F16), sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32), sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64), sym::copysignf128 => ("copysignf128", 2, fx.tcx.types.f128, types::F128), + sym::floorf16 => ("floorf16", 1, fx.tcx.types.f16, types::F16), sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32), sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64), sym::floorf128 => ("floorf128", 1, fx.tcx.types.f128, types::F128), + sym::ceilf16 => ("ceilf16", 1, fx.tcx.types.f16, types::F16), sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32), sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64), sym::ceilf128 => ("ceilf128", 1, fx.tcx.types.f128, types::F128), + sym::truncf16 => ("truncf16", 1, fx.tcx.types.f16, types::F16), sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32), sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64), sym::truncf128 => ("truncf128", 1, fx.tcx.types.f128, types::F128), + sym::round_ties_even_f16 => ("rintf16", 1, fx.tcx.types.f16, types::F16), sym::round_ties_even_f32 => ("rintf", 1, fx.tcx.types.f32, types::F32), sym::round_ties_even_f64 => ("rint", 1, fx.tcx.types.f64, types::F64), sym::round_ties_even_f128 => ("rintf128", 1, fx.tcx.types.f128, types::F128), + sym::roundf16 => ("roundf16", 1, fx.tcx.types.f16, types::F16), sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32), sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64), sym::roundf128 => ("roundf128", 1, fx.tcx.types.f128, types::F128), + sym::sinf16 => ("sinf16", 1, fx.tcx.types.f16, types::F16), sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32), sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64), sym::sinf128 => ("sinf128", 1, fx.tcx.types.f128, types::F128), + sym::cosf16 => ("cosf16", 1, fx.tcx.types.f16, types::F16), sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32), sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64), @@ -489,18 +506,6 @@ fn codegen_float_intrinsic_call<'tcx>( }; CValue::by_val(ret_val, fx.layout_of(ty)) } - sym::powf16 => { - // FIXME(f16_f128): Rust `compiler-builtins` doesn't export `powf16` yet. - let x = codegen_f16_f128::f16_to_f32(fx, args[0]); - let y = codegen_f16_f128::f16_to_f32(fx, args[1]); - let ret_val = fx.lib_call( - "powf", - vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], - vec![AbiParam::new(types::F32)], - &[x, y], - )[0]; - CValue::by_val(codegen_f16_f128::f32_to_f16(fx, ret_val), fx.layout_of(ty)) - } _ => { let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect(); let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0]; From fe2d52dd42251ab5f49896b0ab3411046c45162c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:45:01 +0000 Subject: [PATCH 40/51] Introduce InstanceKind::LlvmIntrinsic This way codegen backends don't have to check for a magic symbol name prefix to detect LLVM intrinsic calls and thus can more easily handle them separately as necessary. --- src/abi/mod.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index ec17e72900dfb..ce780647662b8 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -440,18 +440,6 @@ pub(crate) fn codegen_terminator_call<'tcx>( } } - if fx.tcx.symbol_name(instance).name.starts_with("llvm.") { - crate::intrinsics::codegen_llvm_intrinsic_call( - fx, - fx.tcx.symbol_name(instance).name, - args, - ret_place, - target, - source_info.span, - ); - return; - } - match instance.def { InstanceKind::Intrinsic(_) => { match crate::intrinsics::codegen_intrinsic_call( @@ -466,6 +454,17 @@ pub(crate) fn codegen_terminator_call<'tcx>( Err(instance) => Some(instance), } } + InstanceKind::LlvmIntrinsic(_) => { + crate::intrinsics::codegen_llvm_intrinsic_call( + fx, + fx.tcx.symbol_name(instance).name, + args, + ret_place, + target, + source_info.span, + ); + return; + } // We don't need AsyncDropGlueCtorShim here because it is not `noop func`, // it is `func returning noop future` InstanceKind::Shim(ShimKind::DropGlue(_, None)) => { From b1cc00e25a453cc7cdf88e35fa0140e941735d1b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 15 Jul 2026 17:23:13 +0200 Subject: [PATCH 41/51] Update uses of `rustc_codegen_ssa::errors` --- src/abi/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 478f5d30a255c..491463865b5db 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -14,7 +14,7 @@ use cranelift_codegen::isa::CallConv; use cranelift_module::ModuleError; use rustc_abi::{CanonAbi, ExternAbi, X86Call}; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; -use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall; +use rustc_codegen_ssa::diagnostics::CompilerBuiltinsCannotCall; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; From 3601f44ce6c3c4365c331819735070f0eda78d3f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:48:45 +0200 Subject: [PATCH 42/51] Rustup to rustc 1.99.0-nightly (3d50c25bc 2026-07-16) --- .../0027-stdlib-128bit-atomic-operations.patch | 16 +++++++++------- rust-toolchain.toml | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/patches/0027-stdlib-128bit-atomic-operations.patch b/patches/0027-stdlib-128bit-atomic-operations.patch index 717495cbcdf33..2268ff9cb266e 100644 --- a/patches/0027-stdlib-128bit-atomic-operations.patch +++ b/patches/0027-stdlib-128bit-atomic-operations.patch @@ -37,14 +37,15 @@ diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 8a9a0b5..92ed9a6 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs -@@ -3762,42 +3757,6 @@ atomic_int! { +@@ -3762,44 +3757,6 @@ atomic_int! { 8, u64 AtomicU64 } --#[cfg(target_has_atomic_load_store = "128")] +-#[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { -- cfg(target_has_atomic = "128"), -- cfg(target_has_atomic_primitive_alignment = "128"), +- target_has_atomic_load_store = "128", +- target_has_atomic = "128", +- target_has_atomic_primitive_alignment = "128", - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), @@ -59,10 +60,11 @@ index 8a9a0b5..92ed9a6 100644 - 16, - i128 AtomicI128 -} --#[cfg(target_has_atomic_load_store = "128")] +-#[cfg(any(target_has_atomic_load_store = "128", doc))] -atomic_int! { -- cfg(target_has_atomic = "128"), -- cfg(target_has_atomic_primitive_alignment = "128"), +- target_has_atomic_load_store = "128", +- target_has_atomic = "128", +- target_has_atomic_primitive_alignment = "128", - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 84d3f1a99e086..729ecc656da9e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-07-11" +channel = "nightly-2026-07-17" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From 6c9b08f7f5bc807d7305b7103caa2fcbaf3e148a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:56:52 +0200 Subject: [PATCH 43/51] Fix rustc test suite --- scripts/test_rustc_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index d1ca140664b61..68886b4cc15ed 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -158,6 +158,7 @@ rm -r tests/run-make/missing-unstable-trait-bound # This disables support for un rm -r tests/run-make/const-trait-stable-toolchain # same rm -r tests/run-make/print-request-help-stable-unstable # same rm -r tests/run-make/issue-149402-suggest-unresolve # same +rm -r tests/run-make/const-destruct-stable-toolchain # same rm -r tests/run-make/incr-add-rust-src-component rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path rm -r tests/run-make/export # something about rustc version mismatches From b5e1509b59edbef73c2e741b41cedd5f73d12424 Mon Sep 17 00:00:00 2001 From: kulst Date: Tue, 6 Jan 2026 21:42:06 +0100 Subject: [PATCH 44/51] Treat `-Ctarget-cpu` as a target-modifier when targeting AVR, AMDGCN and NVPTX For AVR, AMDGCN, and NVPTX, crates built with different target CPU values are not generally link-compatible. Add a `requires_consistent_cpu` flag to the target spec and enable it for these targets. When the flag is set, treat `-Ctarget-cpu` as a target modifier and require all linked crates to agree on its value. Reject `-Ctarget-cpu=native` before codegen for targets that set `requires_consistent_cpu` to true. Also do not include `native` in the printed `target-cpus` list for such targets. Add tests covering: - which built-in targets set `requires-consistent-cpu` - cross-crate behavior with and without `requires-consistent-cpu` - that an omitted `-Ctarget-cpu` compares equal to an explicitly specified default CPU - rejection and printing behavior for `native` - precedence of repeated `-Ctarget-cpu` flags in metadata comparison and LLVM IR --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index af783f31a2136..f5c852a2bf970 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,7 +42,7 @@ use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig, back}; use rustc_log::tracing::info; use rustc_middle::dep_graph::WorkProductMap; use rustc_session::Session; -use rustc_session::config::OutputFilenames; +use rustc_session::config::{NATIVE_CPU, OutputFilenames}; use rustc_span::{Symbol, sym}; use rustc_target::spec::{Arch, CfgAbi, Env, Os}; @@ -341,7 +341,7 @@ fn build_isa(sess: &Session, jit: bool) -> Arc { let flags = settings::Flags::new(flags_builder); let isa_builder = match sess.opts.cg.target_cpu.as_deref() { - Some("native") => cranelift_native::builder_with_options(true).unwrap(), + Some(NATIVE_CPU) => cranelift_native::builder_with_options(true).unwrap(), Some(value) => { let mut builder = cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| { From a474ad8a013b0608797ac89876937707470a0d2c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 19 Jul 2026 19:35:17 +0200 Subject: [PATCH 45/51] Rustup to rustc 1.99.0-nightly (eff8269f7 2026-07-18) --- build_system/build_backend.rs | 2 +- build_system/build_sysroot.rs | 2 +- rust-toolchain.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs index 6b14727cd153e..dc977f0ff3fa8 100644 --- a/build_system/build_backend.rs +++ b/build_system/build_backend.rs @@ -43,7 +43,7 @@ pub(crate) fn build_backend( cmd.arg("--release"); - cmd.arg("-Zno-embed-metadata"); + cmd.arg("-Zembed-metadata=no"); eprintln!("[BUILD] rustc_codegen_cranelift"); crate::utils::spawn_and_wait(cmd); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index c3886c1d10464..e4250736e0bb7 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -251,7 +251,7 @@ fn build_clif_sysroot_for_triple( build_cmd.arg("--release"); build_cmd.arg("--features").arg("backtrace panic-unwind"); build_cmd.arg(format!("-Zroot-dir={}", STDLIB_SRC.to_path(dirs).display())); - build_cmd.arg("-Zno-embed-metadata"); + build_cmd.arg("-Zembed-metadata=no"); build_cmd.arg("-Zbuild-dir-new-layout"); build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 729ecc656da9e..7674a2e26ecd0 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-07-17" +channel = "nightly-2026-07-19" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From 0047b65ff8d9ae31bc7068592f17fb9b892311e6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 19 Jul 2026 19:44:09 +0200 Subject: [PATCH 46/51] Fix rustc test suite --- scripts/test_rustc_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 68886b4cc15ed..b83f5f8e4e8b3 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -144,7 +144,7 @@ rm tests/ui/consts/const-mut-refs-crate.rs # same rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift rm -r tests/run-make/short-ice # ICE backtrace begin/end marker mismatch rm tests/ui/codegen/huge-stacks.rs # Cranelift doesn't allow stack frames to exceed 4GB -rm -r tests/run-make/doctests-test_harness # different thread names likely caused by -Zpanic-abort-tests +rm -r tests/run-make/rustdoc/doctest/test_harness # different thread names likely caused by -Zpanic-abort-tests # doesn't work due to the way the rustc test suite is invoked. # should work when using ./x.py test the way it is intended From f35ca5727c8e51b509ef01304a29835ea8f9e2e6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:11:07 +0200 Subject: [PATCH 47/51] Fix rustc bootstrap testing --- scripts/setup_rust_fork.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index ab31c43fb1b12..c2b0e37a8eca7 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -62,6 +62,19 @@ index 2e16f2cf27..3ac3df99a8 100644 # Add RUSTFLAGS_BOOTSTRAP to RUSTFLAGS for bootstrap compilation. # Note that RUSTFLAGS_BOOTSTRAP should always be added to the end of # RUSTFLAGS, since that causes RUSTFLAGS_BOOTSTRAP to override RUSTFLAGS. +diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs +index 6de70c7d70c..d5035b581ce 100644 +--- a/src/bootstrap/src/core/builder/cargo.rs ++++ b/src/bootstrap/src/core/builder/cargo.rs +@@ -1197,7 +1197,7 @@ fn cargo( + cargo.env("RUSTC_BOOTSTRAP", "1"); + + if matches!(mode, Mode::Std) { +- cargo.arg("-Zno-embed-metadata"); ++ cargo.arg("-Zembed-metadata=no"); + } + + if self.config.dump_bootstrap_shims { diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index bc68bfe396..00143ef3ed 100644 --- a/src/bootstrap/src/core/config/config.rs From d28bb795fd60e0d52766772bc1a3fc9908251464 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:25:19 +0200 Subject: [PATCH 48/51] Update to Cranelift 0.134 --- Cargo.lock | 84 +++++++++++++++++----------------- Cargo.toml | 24 +++++----- src/abi/mod.rs | 2 +- src/allocator.rs | 2 +- src/base.rs | 14 +++--- src/cast.rs | 6 +-- src/codegen_f16_f128.rs | 22 ++++----- src/common.rs | 20 ++++---- src/constant.rs | 7 +-- src/intrinsics/llvm_aarch64.rs | 8 ++-- src/intrinsics/llvm_x86.rs | 36 +++++++-------- src/intrinsics/mod.rs | 34 +++++++++----- src/intrinsics/simd.rs | 15 +++--- src/main_shim.rs | 2 +- src/num.rs | 27 ++++++----- src/pointer.rs | 10 ++-- src/unsize.rs | 4 +- src/value_and_place.rs | 9 ++-- 18 files changed, 173 insertions(+), 153 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 78abf73d36a5c..88ea75a6b0299 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arbitrary" @@ -43,27 +43,27 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cranelift-assembler-x64" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715783c05f20985a5dfe6bfdccbfcb146cb44bfd8f6ff1d09526c3bf442fdac5" +checksum = "a25c5b1bb1d86ae68dca1826a1b668b9951056c42e0eef89e35942c7001eb481" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3da5a783f2b72af39ab98c1bf3157e081511e1febeafe550d71a4d0ba75f66" +checksum = "c1b052a1fd94b4565697c56f9d358723c7c354feef42a6569081225c1bed6e86" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2e9b7adf77fa02204d4d523ae4f171b6591c2632b030865336335c7d7b420e" +checksum = "a599e8ffa6a1a118d2dce4809073ad2f2534b1ae1c6d55697b6b8a324446c971" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -71,18 +71,18 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f09d9f397eae612ac15becf0e0b2165d2232003f4f2a1549572a724d1c48c5" +checksum = "1e1ae13182fadc731b1387287b4c6d257b9191fbb7c980ebb2d74c2db74b743e" dependencies = [ "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e004cf1270abc82f7b9fb32d1a9d73a1cc0d5a0215b97db8c32658748e78b01" +checksum = "dce9c62e5e11d12f4a8f5f939f29899f32002d38193b47491e2539705a1d447e" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f32034641f96b123e4fdb5666e726d9f252e222638fc8fdd905b4ff18c3c4e" +checksum = "3c29d6b53cfd758dc1a269310fc92199056db03943b9b774b4911cd615e02ad7" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -119,24 +119,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746566ae868b0e87a89206b3856350886a4fc2e078ce6485bec2ea6727c31685" +checksum = "8cd8cfe23f3349bfa62e06a450cd94cf5f84fbe78556f4df3276534c4b93d834" [[package]] name = "cranelift-control" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "def01ab5cd08a1be551d4bc96adc6af91f89138c4f81d0a60fdf3b9f82272703" +checksum = "f991b4712f21d9502758f7e6cb11746ca2e6186b06c42995eaf31b1e50a9c498" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "341d5e1e071320505ebbc8194a8eb61fa94394b1ed93ba3cbec3be5fa1c3c1ce" +checksum = "5e24f9c233a43730e22cf42ab0c03f41265e147eca97b3362be01e0da359d7a5" dependencies = [ "cranelift-bitset", "wasmtime-internal-core", @@ -144,9 +144,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97c6f3e2419ecb54a5503994d35421554c5e487d0493bc86ea96907bab51fd29" +checksum = "61325401051d35ddbf8e2b5d5b5dec8dd54688c9054046b62e76146e61c88a99" dependencies = [ "cranelift-codegen", "hashbrown 0.17.0", @@ -157,15 +157,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21aa47a5e0b1e9fb2c9348459088d5dbb872ebe17781ada1270d8364c7e73257" +checksum = "62f71d642812aa70284750719efbd2cc5ad7e7c5e6c43c026b5b0686ce8a6052" [[package]] name = "cranelift-jit" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0401e1c72d772c397726f49b29f048fe397fe6b1f761283c4bfa1f9ebe6cb86b" +checksum = "92a0f9300c8a5738553a440fe4f25b85f574d5c41b946880dabd47c809f87997" dependencies = [ "anyhow", "cranelift-codegen", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a1992fd5ae19ac2a8e8f37b134a797a7a3bde9ee9d3117d560ad382b985dbef" +checksum = "5f723983af35e690c4e2fc160df5d8895bb4a419a54d693367163d781bd830cc" dependencies = [ "anyhow", "cranelift-codegen", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3054a03ac285b170662ec9530602b01cd394a7428cd753b48d9cae51f05249a" +checksum = "b5c50cffac70c30cc5d0b1d8d287e7712dc76b1181c43697de63d27f38236ddd" dependencies = [ "cranelift-codegen", "libc", @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2adff988bc0579a7233517ce0a9276c61cca2ec7e2e56e83226a5b32e35653b" +checksum = "430e18f4886ba98f6a7dbb82c86df19f28f27cac417611e5dcc427a7a8decac9" dependencies = [ "anyhow", "cranelift-codegen", @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.133.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd2498406acfcfd39c69078af187154ac8dcd31d3082888b258deb2a4fd0ecf4" +checksum = "bc5972dc0a5df7ba6f11e8e1654287c40ea1704ca6ce3b76ef54fa58445ee4ad" [[package]] name = "crc32fast" @@ -340,9 +340,9 @@ checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" -version = "0.2.3" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723e3ebdcdc5c023db1df315364573789f8857c11b631a2fdfad7c00f5c046b4" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] @@ -481,9 +481,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "unicode-ident" @@ -493,9 +493,9 @@ checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "wasmtime-internal-core" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f6ce74d60a8ed870548e7efa9710c54f982bbfcf80e5ee5eee8498318616483" +checksum = "852f3326264aea590131a422e3ecb33fae3b6695fd67e2e202e2799cd8d27629" dependencies = [ "hashbrown 0.17.0", "libm", @@ -503,9 +503,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bba33d9d951a9a974a866e80c864a9a46b28086e369582e0caa78e14a9f29e4" +checksum = "e537f59b5ab127ca2d421ecbd8728d70bd6cdc9891cec193fb1f594c3309836c" dependencies = [ "cfg-if", "libc", diff --git a/Cargo.toml b/Cargo.toml index aa43e43d2165a..cb595332bac61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,12 +8,12 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.133.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } -cranelift-frontend = { version = "0.133.0" } -cranelift-module = { version = "0.133.0" } -cranelift-native = { version = "0.133.0" } -cranelift-jit = { version = "0.133.0", optional = true } -cranelift-object = { version = "0.133.0", default-features = false } +cranelift-codegen = { version = "0.134.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } +cranelift-frontend = { version = "0.134.0" } +cranelift-module = { version = "0.134.0" } +cranelift-native = { version = "0.134.0" } +cranelift-jit = { version = "0.134.0", optional = true } +cranelift-object = { version = "0.134.0", default-features = false } target-lexicon = "0.13" gimli = { version = "0.33", default-features = false, features = ["write"] } object = { version = "0.39.1", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } @@ -24,12 +24,12 @@ smallvec = "1.8.1" # Uncomment to use an unreleased version of cranelift #[patch.crates-io] -#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } -#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } -#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } -#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } -#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } -#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-46.0.0" } +#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-47.0.0" } +#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-47.0.0" } +#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-47.0.0" } +#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-47.0.0" } +#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-47.0.0" } +#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-47.0.0" } # Uncomment to use local checkout of cranelift #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 2ba569a1b85a0..5e152af835613 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -756,7 +756,7 @@ pub(crate) fn codegen_drop<'tcx>( let ptr = ptr.get_addr(fx); let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); - let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0); + let is_null = fx.bcx.ins().icmp_imm_u(IntCC::Equal, drop_fn, 0); let target_block = fx.get_block(target); let continued = fx.bcx.create_block(); fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]); diff --git a/src/allocator.rs b/src/allocator.rs index 3c18748ee24b4..9133fd738a9d2 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -73,7 +73,7 @@ pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module, methods: &[Alloc bcx.switch_to_block(block); bcx.ins().return_(&[]); bcx.seal_all_blocks(); - bcx.finalize(); + bcx.finalize(module.target_config()); module.define_function(func_id, &mut ctx).unwrap(); } diff --git a/src/base.rs b/src/base.rs index 0e89e9516e1b7..451983a9053b8 100644 --- a/src/base.rs +++ b/src/base.rs @@ -120,7 +120,7 @@ pub(crate) fn codegen_fn<'tcx>( tcx.prof.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx, start_block)); fx.bcx.seal_all_blocks(); - fx.bcx.finalize(); + fx.bcx.finalize(fx.module.target_config()); // Recover all necessary data from fx, before accessing func will prevent future access to it. let symbol_name = fx.symbol_name; @@ -667,7 +667,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let val = operand.load_scalar(fx); match layout.ty.kind() { ty::Bool => { - let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0); + let res = fx.bcx.ins().icmp_imm_u(IntCC::Equal, val, 0); CValue::by_val(res, layout) } ty::Uint(_) | ty::Int(_) => { @@ -855,13 +855,13 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: fx.bcx.ins().jump(loop_block, &[zero.into()]); fx.bcx.switch_to_block(loop_block); - let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64); + let done = fx.bcx.ins().icmp_imm_u(IntCC::Equal, index, times as i64); fx.bcx.ins().brif(done, done_block, &[], loop_block2, &[]); fx.bcx.switch_to_block(loop_block2); let to = lval.place_index(fx, index); to.write_cvalue(fx, operand); - let index = fx.bcx.ins().iadd_imm(index, 1); + let index = fx.bcx.ins().iadd_imm_u(index, 1); fx.bcx.ins().jump(loop_block, &[index.into()]); fx.bcx.switch_to_block(done_block); @@ -953,7 +953,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let count = codegen_operand(fx, count).load_scalar(fx); let bytes = if elem_size != 1 { - fx.bcx.ins().imul_imm(count, elem_size as i64) + fx.bcx.ins().imul_imm_u(count, elem_size as i64) } else { count }; @@ -1006,7 +1006,7 @@ pub(crate) fn codegen_place<'tcx>( fx.bcx.ins().iconst(fx.pointer_type, offset as i64) } else { let len = codegen_array_len(fx, cplace); - fx.bcx.ins().iadd_imm(len, -(offset as i64)) + fx.bcx.ins().iadd_imm_s(len, -(offset as i64)) }; cplace = cplace.place_index(fx, index); } @@ -1033,7 +1033,7 @@ pub(crate) fn codegen_place<'tcx>( let (ptr, len) = cplace.to_ptr_unsized(); cplace = CPlace::for_ptr_with_extra( ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), - fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), + fx.bcx.ins().iadd_imm_s(len, -(from as i64 + to as i64)), cplace.layout(), ); } diff --git a/src/cast.rs b/src/cast.rs index 8a725680e7059..f124739d1e154 100644 --- a/src/cast.rs +++ b/src/cast.rs @@ -133,12 +133,12 @@ pub(crate) fn clif_int_or_float_cast( let max_val = fx.bcx.ins().iconst(types::I32, max); let val = if to_signed { - let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, min); - let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, val, max); + let has_underflow = fx.bcx.ins().icmp_imm_s(IntCC::SignedLessThan, val, min); + let has_overflow = fx.bcx.ins().icmp_imm_s(IntCC::SignedGreaterThan, val, max); let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, val); fx.bcx.ins().select(has_overflow, max_val, bottom_capped) } else { - let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, val, max); + let has_overflow = fx.bcx.ins().icmp_imm_u(IntCC::UnsignedGreaterThan, val, max); fx.bcx.ins().select(has_overflow, max_val, val) }; fx.bcx.ins().ireduce(to_ty, val) diff --git a/src/codegen_f16_f128.rs b/src/codegen_f16_f128.rs index 7a386db0cd6a3..089ea87d74f6c 100644 --- a/src/codegen_f16_f128.rs +++ b/src/codegen_f16_f128.rs @@ -148,28 +148,28 @@ pub(crate) fn codegen_f128_binop( pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value); - let bits = fx.bcx.ins().bxor_imm(bits, 0x8000); + let bits = fx.bcx.ins().bxor_imm_u(bits, 0x8000); fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits) } pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value); let (low, high) = fx.bcx.ins().isplit(bits); - let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64); + let high = fx.bcx.ins().bxor_imm_u(high, 0x8000_0000_0000_0000_u64 as i64); let bits = fx.bcx.ins().iconcat(low, high); fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) } pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value); - let bits = fx.bcx.ins().band_imm(bits, 0x7fff); + let bits = fx.bcx.ins().band_imm_u(bits, 0x7fff); fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits) } pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value); let (low, high) = fx.bcx.ins().isplit(bits); - let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64); + let high = fx.bcx.ins().band_imm_u(high, 0x7fff_ffff_ffff_ffff_u64 as i64); let bits = fx.bcx.ins().iconcat(low, high); fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) } @@ -177,8 +177,8 @@ pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { let lhs = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), lhs); let rhs = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), rhs); - let res = fx.bcx.ins().band_imm(lhs, 0x7fff); - let sign = fx.bcx.ins().band_imm(rhs, 0x8000); + let res = fx.bcx.ins().band_imm_u(lhs, 0x7fff); + let sign = fx.bcx.ins().band_imm_u(rhs, 0x8000); let res = fx.bcx.ins().bor(res, sign); fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), res) } @@ -188,8 +188,8 @@ pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Va let rhs = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), rhs); let (low, lhs_high) = fx.bcx.ins().isplit(lhs); let (_, rhs_high) = fx.bcx.ins().isplit(rhs); - let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64); - let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64); + let high = fx.bcx.ins().band_imm_u(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64); + let sign = fx.bcx.ins().band_imm_u(rhs_high, 0x8000_0000_0000_0000_u64 as i64); let high = fx.bcx.ins().bor(high, sign); let res = fx.bcx.ins().iconcat(low, high); fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), res) @@ -275,12 +275,12 @@ pub(crate) fn codegen_cast( let max_val = fx.bcx.ins().iconst(types::I32, max); let val = if to_signed { - let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, ret, min); - let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, ret, max); + let has_underflow = fx.bcx.ins().icmp_imm_s(IntCC::SignedLessThan, ret, min); + let has_overflow = fx.bcx.ins().icmp_imm_s(IntCC::SignedGreaterThan, ret, max); let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret); fx.bcx.ins().select(has_overflow, max_val, bottom_capped) } else { - let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, ret, max); + let has_overflow = fx.bcx.ins().icmp_imm_u(IntCC::UnsignedGreaterThan, ret, max); fx.bcx.ins().select(has_overflow, max_val, ret) }; fx.bcx.ins().ireduce(to_ty, val) diff --git a/src/common.rs b/src/common.rs index 283e7765af216..1bdb3efefa1aa 100644 --- a/src/common.rs +++ b/src/common.rs @@ -116,13 +116,13 @@ pub(crate) fn codegen_icmp_imm( match intcc { IntCC::Equal => { - let lsb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_lsb, rhs_lsb); - let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb); + let lsb_eq = fx.bcx.ins().icmp_imm_u(IntCC::Equal, lhs_lsb, rhs_lsb); + let msb_eq = fx.bcx.ins().icmp_imm_u(IntCC::Equal, lhs_msb, rhs_msb); fx.bcx.ins().band(lsb_eq, msb_eq) } IntCC::NotEqual => { - let lsb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_lsb, rhs_lsb); - let msb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_msb, rhs_msb); + let lsb_ne = fx.bcx.ins().icmp_imm_u(IntCC::NotEqual, lhs_lsb, rhs_lsb); + let msb_ne = fx.bcx.ins().icmp_imm_u(IntCC::NotEqual, lhs_msb, rhs_msb); fx.bcx.ins().bor(lsb_ne, msb_ne) } _ => { @@ -132,16 +132,16 @@ pub(crate) fn codegen_icmp_imm( // msb_cc // } - let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb); - let lsb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_lsb, rhs_lsb); - let msb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_msb, rhs_msb); + let msb_eq = fx.bcx.ins().icmp_imm_u(IntCC::Equal, lhs_msb, rhs_msb); + let lsb_cc = fx.bcx.ins().icmp_imm_u(intcc, lhs_lsb, rhs_lsb); + let msb_cc = fx.bcx.ins().icmp_imm_u(intcc, lhs_msb, rhs_msb); fx.bcx.ins().select(msb_eq, lsb_cc, msb_cc) } } } else { let rhs = rhs as i64; // Truncates on purpose in case rhs is actually an unsigned value - fx.bcx.ins().icmp_imm(intcc, lhs, rhs) + fx.bcx.ins().icmp_imm_u(intcc, lhs, rhs) } } @@ -262,7 +262,7 @@ pub(crate) fn create_wrapper_function( bcx.ins().return_(&results); bcx.seal_all_blocks(); - bcx.finalize(); + bcx.finalize(module.target_config()); } module.define_function(wrapper_func_id, &mut ctx).unwrap(); } @@ -400,7 +400,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { key: None, }); let base_ptr = self.bcx.ins().stack_addr(self.pointer_type, stack_slot, 0); - let misalign_offset = self.bcx.ins().band_imm(base_ptr, i64::from(align - 1)); + let misalign_offset = self.bcx.ins().band_imm_u(base_ptr, i64::from(align - 1)); let align = self.bcx.ins().iconst(self.pointer_type, i64::from(align)); let realign_offset = self.bcx.ins().isub(align, misalign_offset); Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset)) diff --git a/src/constant.rs b/src/constant.rs index aadddb08bd86d..175ca37a3b74e 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -206,9 +206,10 @@ pub(crate) fn codegen_const_value<'tcx>( } }; let val = if offset.bytes() != 0 { - fx.bcx - .ins() - .iadd_imm(base_addr, fx.tcx.truncate_to_target_usize(offset.bytes()) as i64) + fx.bcx.ins().iadd_imm_u( + base_addr, + fx.tcx.truncate_to_target_usize(offset.bytes()) as i64, + ) } else { base_addr }; diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index 87044211d234d..b046bf1ee4df9 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -422,7 +422,7 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( for i in 0..8 { let idx_lane = idx.value_lane(fx, i).load_scalar(fx); let is_zero = - fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, idx_lane, 16); + fx.bcx.ins().icmp_imm_u(IntCC::UnsignedGreaterThanOrEqual, idx_lane, 16); let t_idx = fx.bcx.ins().uextend(fx.pointer_type, idx_lane); let t_lane = t.value_lane_dyn(fx, t_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, t_lane); @@ -436,7 +436,7 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( for i in 0..16 { let idx_lane = idx.value_lane(fx, i).load_scalar(fx); let is_zero = - fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, idx_lane, 16); + fx.bcx.ins().icmp_imm_u(IntCC::UnsignedGreaterThanOrEqual, idx_lane, 16); let t_idx = fx.bcx.ins().uextend(fx.pointer_type, idx_lane); let t_lane = t.value_lane_dyn(fx, t_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, t_lane); @@ -1035,7 +1035,7 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( b, ret, &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| { - let rot = fx.bcx.ins().rotl_imm(b_lane, 1); + let rot = fx.bcx.ins().rotl_imm_u(b_lane, 1); fx.bcx.ins().bxor(a_lane, rot) }, ); @@ -1162,7 +1162,7 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( let a_lane = fx.bcx.ins().sextend(product_ty, a_lane); let b_lane = fx.bcx.ins().sextend(product_ty, b_lane); let product = fx.bcx.ins().imul(a_lane, b_lane); - let product = fx.bcx.ins().sshr_imm(product, shift); + let product = fx.bcx.ins().sshr_imm_u(product, shift); let max = fx.bcx.ins().iconst(product_ty, max); let result = fx.bcx.ins().smin(product, max); fx.bcx.ins().ireduce(result_ty, result) diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs index 29303f701588e..136e98964bc35 100644 --- a/src/intrinsics/llvm_x86.rs +++ b/src/intrinsics/llvm_x86.rs @@ -130,10 +130,10 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let mask_lane = match mask_lane_clif_ty { types::I32 | types::F32 => { - fx.bcx.ins().band_imm(mask_lane, 0x8000_0000u64 as i64) + fx.bcx.ins().band_imm_u(mask_lane, 0x8000_0000u64 as i64) } types::I64 | types::F64 => { - fx.bcx.ins().band_imm(mask_lane, 0x8000_0000_0000_0000u64 as i64) + fx.bcx.ins().band_imm_u(mask_lane, 0x8000_0000_0000_0000u64 as i64) } _ => unreachable!(), }; @@ -324,8 +324,8 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let zero = fx.bcx.ins().iconst(types::I8, 0); for i in 0..16 { let b_lane = b.value_lane(fx, i).load_scalar(fx); - let is_zero = fx.bcx.ins().band_imm(b_lane, 0x80); - let a_idx = fx.bcx.ins().band_imm(b_lane, 0xf); + let is_zero = fx.bcx.ins().band_imm_u(b_lane, 0x80); + let a_idx = fx.bcx.ins().band_imm_u(b_lane, 0xf); let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, a_lane); @@ -335,9 +335,9 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( if intrinsic == "llvm.x86.avx2.pshuf.b" { for i in 16..32 { let b_lane = b.value_lane(fx, i).load_scalar(fx); - let is_zero = fx.bcx.ins().band_imm(b_lane, 0x80); - let b_lane_masked = fx.bcx.ins().band_imm(b_lane, 0xf); - let a_idx = fx.bcx.ins().iadd_imm(b_lane_masked, 16); + let is_zero = fx.bcx.ins().band_imm_u(b_lane, 0x80); + let b_lane_masked = fx.bcx.ins().band_imm_u(b_lane, 0xf); + let a_idx = fx.bcx.ins().iadd_imm_u(b_lane_masked, 16); let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); let res = fx.bcx.ins().select(is_zero, zero, a_lane); @@ -388,9 +388,9 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( b_low: Value, control: Value, ) -> Value { - let a_or_b = fx.bcx.ins().band_imm(control, 0b0010); - let high_or_low = fx.bcx.ins().band_imm(control, 0b0001); - let is_zero = fx.bcx.ins().band_imm(control, 0b1000); + let a_or_b = fx.bcx.ins().band_imm_u(control, 0b0010); + let high_or_low = fx.bcx.ins().band_imm_u(control, 0b0001); + let is_zero = fx.bcx.ins().band_imm_u(control, 0b1000); let zero = fx.bcx.ins().iconst(types::I64, 0); let zero = fx.bcx.ins().iconcat(zero, zero); @@ -404,7 +404,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let control0 = imm8; let res_low = select4(fx, a_high, a_low, b_high, b_low, control0); - let control1 = fx.bcx.ins().ushr_imm(imm8, 4); + let control1 = fx.bcx.ins().ushr_imm_u(imm8, 4); let res_high = select4(fx, a_high, a_low, b_high, b_low, control1); ret.place_typed_lane(fx, fx.tcx.types.u128, 0).to_ptr().store( @@ -497,8 +497,8 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let a_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), a_lane); let b_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), b_lane); let sum = fx.bcx.ins().iadd(a_lane, b_lane); - let num_plus_one = fx.bcx.ins().iadd_imm(sum, 1); - let res = fx.bcx.ins().ushr_imm(num_plus_one, 1); + let num_plus_one = fx.bcx.ins().iadd_imm_u(sum, 1); + let res = fx.bcx.ins().ushr_imm_u(num_plus_one, 1); fx.bcx.ins().ireduce(lane_ty, res) }, ); @@ -579,7 +579,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let (val, has_overflow) = fx.bcx.ins().sadd_overflow(mul0, mul1); - let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, mul1, 0); + let rhs_ge_zero = fx.bcx.ins().icmp_imm_s(IntCC::SignedGreaterThanOrEqual, mul1, 0); let min = fx.bcx.ins().iconst(types::I16, i64::from(i16::MIN as u16)); let max = fx.bcx.ins().iconst(types::I16, i64::from(i16::MAX as u16)); @@ -649,9 +649,9 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let b_lane = fx.bcx.ins().sextend(types::I32, b_lane); let mul: Value = fx.bcx.ins().imul(a_lane, b_lane); - let shifted = fx.bcx.ins().ushr_imm(mul, 14); - let incremented = fx.bcx.ins().iadd_imm(shifted, 1); - let shifted_again = fx.bcx.ins().ushr_imm(incremented, 1); + let shifted = fx.bcx.ins().ushr_imm_u(mul, 14); + let incremented = fx.bcx.ins().iadd_imm_u(shifted, 1); + let shifted_again = fx.bcx.ins().ushr_imm_u(incremented, 1); let res_lane = fx.bcx.ins().ireduce(types::I16, shifted_again); let res_lane = CValue::by_val(res_lane, ret_lane_layout); @@ -1279,7 +1279,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( let all_zero1 = fx.bcx.ins().bor(zero2, zero3); let all_zero = fx.bcx.ins().bor(all_zero0, all_zero1); - let res = fx.bcx.ins().icmp_imm(IntCC::Equal, all_zero, 0); + let res = fx.bcx.ins().icmp_imm_u(IntCC::Equal, all_zero, 0); let res = CValue::by_val( fx.bcx.ins().uextend(types::I32, res), fx.layout_of(fx.tcx.types.i32), diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 292c8004ef434..3a8adc261e976 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -229,10 +229,10 @@ fn simd_reduce_bool<'tcx>( assert!(ret.layout().ty.is_bool()); let res_val = val.value_lane(fx, 0).load_scalar(fx); - let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean + let mut res_val = fx.bcx.ins().band_imm_u(res_val, 1); // mask to boolean for lane_idx in 1..lane_count { let lane = val.value_lane(fx, lane_idx).load_scalar(fx); - let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean + let lane = fx.bcx.ins().band_imm_u(lane, 1); // mask to boolean res_val = f(fx, res_val, lane); } let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 { @@ -551,8 +551,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let elem_ty = generic_args.type_at(0); let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); assert_eq!(args.len(), 3); - let byte_amount = - if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; + let byte_amount = if elem_size != 1 { + fx.bcx.ins().imul_imm_u(count, elem_size as i64) + } else { + count + }; // FIXME emit_small_memmove fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount); @@ -567,8 +570,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let elem_ty = generic_args.type_at(0); let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); assert_eq!(args.len(), 3); - let byte_amount = - if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; + let byte_amount = if elem_size != 1 { + fx.bcx.ins().imul_imm_u(count, elem_size as i64) + } else { + count + }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} if intrinsic == sym::volatile_copy_nonoverlapping_memory { @@ -672,7 +678,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let pointee_ty = base.layout().ty.builtin_deref(true).unwrap(); let pointee_size = fx.layout_of(pointee_ty).size.bytes(); let ptr_diff = if pointee_size != 1 { - fx.bcx.ins().imul_imm(offset, pointee_size as i64) + fx.bcx.ins().imul_imm_u(offset, pointee_size as i64) } else { offset }; @@ -698,7 +704,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap(); let pointee_size = fx.layout_of(pointee_ty).size.bytes(); let count = if pointee_size != 1 { - fx.bcx.ins().imul_imm(count, pointee_size as i64) + fx.bcx.ins().imul_imm_u(count, pointee_size as i64) } else { count }; @@ -832,10 +838,16 @@ fn codegen_regular_intrinsic_call<'tcx>( let usize_layout = fx.layout_of(fx.tcx.types.usize); // Because diff_bytes ULE isize::MAX, this would be fine as signed, // but unsigned is slightly easier to codegen, so might as well. - CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout) + CValue::by_val( + fx.bcx.ins().udiv_imm_u(diff_bytes, pointee_size as i64), + usize_layout, + ) } else { let isize_layout = fx.layout_of(fx.tcx.types.isize); - CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout) + CValue::by_val( + fx.bcx.ins().sdiv_imm_u(diff_bytes, pointee_size as i64), + isize_layout, + ) }; ret.write_cvalue(fx, val); } @@ -1506,7 +1518,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let returns = vec![AbiParam::new(types::I32)]; let args = &[lhs_ref, rhs_ref, bytes_val]; let cmp = fx.lib_call("memcmp", params, returns, args)[0]; - fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0) + fx.bcx.ins().icmp_imm_u(IntCC::Equal, cmp, 0) }; ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout())); } diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index e118447df4810..b754a923dbfe8 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -815,7 +815,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let a_lane = a.value_lane(fx, lane).load_scalar(fx); let b_lane = b.value_lane(fx, lane).load_scalar(fx); - let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0); + let m_lane = fx.bcx.ins().icmp_imm_u(IntCC::Equal, m_lane, 0); let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); @@ -874,12 +874,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( Endian::Big => lane_count - 1 - lane, Endian::Little => lane, }; - let m_lane = fx.bcx.ins().ushr_imm(m, mask_lane.cast_signed()); - let m_lane = fx.bcx.ins().band_imm(m_lane, 1); + let m_lane = fx.bcx.ins().ushr_imm_u(m, mask_lane.cast_signed()); + let m_lane = fx.bcx.ins().band_imm_u(m_lane, 1); let a_lane = a.value_lane(fx, lane).load_scalar(fx); let b_lane = b.value_lane(fx, lane).load_scalar(fx); - let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0); + let m_lane = fx.bcx.ins().icmp_imm_u(IntCC::Equal, m_lane, 0); let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); @@ -933,11 +933,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let a_lane = a.value_lane(fx, lane).load_scalar(fx); // extract sign bit of an int - let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_clif_ty.bits() - 1)); + let a_lane_sign = + fx.bcx.ins().ushr_imm_u(a_lane, i64::from(lane_clif_ty.bits() - 1)); // shift sign bit into result let a_lane_sign = clif_intcast(fx, a_lane_sign, res_type, false); - res = fx.bcx.ins().ishl_imm(res, 1); + res = fx.bcx.ins().ishl_imm_u(res, 1); res = fx.bcx.ins().bor(res, a_lane_sign); } @@ -1002,7 +1003,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let offset_lane = offset.value_lane(fx, lane_idx).load_scalar(fx); let ptr_diff = if pointee_size != 1 { - fx.bcx.ins().imul_imm(offset_lane, pointee_size as i64) + fx.bcx.ins().imul_imm_u(offset_lane, pointee_size as i64) } else { offset_lane }; diff --git a/src/main_shim.rs b/src/main_shim.rs index dd776f9cfcaee..109933f4d8556 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -156,7 +156,7 @@ pub(crate) fn maybe_create_entry_wrapper( bcx.ins().return_(&[result]); bcx.seal_all_blocks(); - bcx.finalize(); + bcx.finalize(m.target_config()); } if let Err(err) = m.define_function(cmain_func_id, &mut ctx) { diff --git a/src/num.rs b/src/num.rs index e533c0b631b01..f1c44df1f6886 100644 --- a/src/num.rs +++ b/src/num.rs @@ -218,7 +218,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let has_overflow = if !signed { fx.bcx.ins().icmp(IntCC::UnsignedLessThan, val, lhs) } else { - let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0); + let rhs_is_negative = fx.bcx.ins().icmp_imm_s(IntCC::SignedLessThan, rhs, 0); let slt = fx.bcx.ins().icmp(IntCC::SignedLessThan, val, lhs); fx.bcx.ins().bxor(rhs_is_negative, slt) }; @@ -232,7 +232,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let has_overflow = if !signed { fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, val, lhs) } else { - let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0); + let rhs_is_negative = fx.bcx.ins().icmp_imm_s(IntCC::SignedLessThan, rhs, 0); let sgt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, val, lhs); fx.bcx.ins().bxor(rhs_is_negative, sgt) }; @@ -249,7 +249,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let lhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), lhs); let rhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), rhs); let val = fx.bcx.ins().imul(lhs, rhs); - let has_overflow = fx.bcx.ins().icmp_imm( + let has_overflow = fx.bcx.ins().icmp_imm_u( IntCC::UnsignedGreaterThan, val, (1 << ty.bits()) - 1, @@ -261,9 +261,12 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let lhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), lhs); let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs); let val = fx.bcx.ins().imul(lhs, rhs); - let has_underflow = - fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1))); - let has_overflow = fx.bcx.ins().icmp_imm( + let has_underflow = fx.bcx.ins().icmp_imm_s( + IntCC::SignedLessThan, + val, + -(1 << (ty.bits() - 1)), + ); + let has_overflow = fx.bcx.ins().icmp_imm_s( IntCC::SignedGreaterThan, val, (1 << (ty.bits() - 1)) - 1, @@ -275,7 +278,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let val = fx.bcx.ins().imul(lhs, rhs); let has_overflow = if !signed { let val_hi = fx.bcx.ins().umulhi(lhs, rhs); - fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0) + fx.bcx.ins().icmp_imm_u(IntCC::NotEqual, val_hi, 0) } else { // Based on LLVM's instruction sequence for compiling // a.checked_mul(b).is_some() to riscv64gc: @@ -285,9 +288,9 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( // xor a0, a0, a2 // snez a0, a0 let val_hi = fx.bcx.ins().smulhi(lhs, rhs); - let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1)); + let val_sign = fx.bcx.ins().sshr_imm_u(val, i64::from(ty.bits() - 1)); let xor = fx.bcx.ins().bxor(val_hi, val_sign); - fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0) + fx.bcx.ins().icmp_imm_u(IntCC::NotEqual, xor, 0) }; (val, has_overflow) } @@ -324,13 +327,13 @@ pub(crate) fn codegen_saturating_int_binop<'tcx>( (BinOp::Sub, false) => fx.bcx.ins().select(has_overflow, min, val), (BinOp::Add, true) => { let rhs = rhs.load_scalar(fx); - let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let rhs_ge_zero = fx.bcx.ins().icmp_imm_s(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); fx.bcx.ins().select(has_overflow, sat_val, val) } (BinOp::Sub, true) => { let rhs = rhs.load_scalar(fx); - let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let rhs_ge_zero = fx.bcx.ins().icmp_imm_s(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); fx.bcx.ins().select(has_overflow, sat_val, val) } @@ -449,7 +452,7 @@ fn codegen_ptr_binop<'tcx>( let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap(); let (base, offset) = (in_lhs, in_rhs.load_scalar(fx)); let pointee_size = fx.layout_of(pointee_ty).size.bytes(); - let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64); + let ptr_diff = fx.bcx.ins().imul_imm_u(offset, pointee_size as i64); let base_val = base.load_scalar(fx); let res = fx.bcx.ins().iadd(base_val, ptr_diff); CValue::by_val(res, base.layout()) diff --git a/src/pointer.rs b/src/pointer.rs index 56ea1096c770b..b9da3ab6c8d40 100644 --- a/src/pointer.rs +++ b/src/pointer.rs @@ -41,7 +41,7 @@ impl Pointer { match self.base { PointerBase::Addr(base_addr) => { let offset: i64 = self.offset.into(); - if offset == 0 { base_addr } else { fx.bcx.ins().iadd_imm(base_addr, offset) } + if offset == 0 { base_addr } else { fx.bcx.ins().iadd_imm_s(base_addr, offset) } } PointerBase::Stack(stack_slot) => { fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset) @@ -71,7 +71,7 @@ impl Pointer { fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()) } }; - let addr = fx.bcx.ins().iadd_imm(base_addr, new_offset); + let addr = fx.bcx.ins().iadd_imm_s(base_addr, new_offset); Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) } } else { panic!( @@ -114,7 +114,9 @@ impl Pointer { ) -> Value { match self.base { PointerBase::Addr(base_addr) => fx.bcx.ins().load(ty, flags, base_addr, self.offset), - PointerBase::Stack(stack_slot) => fx.bcx.ins().stack_load(ty, stack_slot, self.offset), + PointerBase::Stack(stack_slot) => { + fx.bcx.ins().stack_load(fx.pointer_type, ty, stack_slot, self.offset) + } PointerBase::Dangling(_align) => unreachable!(), } } @@ -125,7 +127,7 @@ impl Pointer { fx.bcx.ins().store(flags, value, base_addr, self.offset); } PointerBase::Stack(stack_slot) => { - fx.bcx.ins().stack_store(value, stack_slot, self.offset); + fx.bcx.ins().stack_store(fx.pointer_type, value, stack_slot, self.offset); } PointerBase::Dangling(_align) => unreachable!(), } diff --git a/src/unsize.rs b/src/unsize.rs index 3dbb689cccd26..48fb0f6c7d4a8 100644 --- a/src/unsize.rs +++ b/src/unsize.rs @@ -190,7 +190,7 @@ pub(crate) fn size_and_align_of<'tcx>( // The info in this case is the length of the str, so the size is that // times the unit size. ( - fx.bcx.ins().imul_imm(info.unwrap(), unit.size.bytes() as i64), + fx.bcx.ins().imul_imm_u(info.unwrap(), unit.size.bytes() as i64), fx.bcx.ins().iconst(fx.pointer_type, unit.align.bytes() as i64), ) } @@ -282,7 +282,7 @@ pub(crate) fn size_and_align_of<'tcx>( // emulated via the semi-standard fast bit trick: // // `(size + (align-1)) & -align` - let addend = fx.bcx.ins().iadd_imm(full_align, -1); + let addend = fx.bcx.ins().iadd_imm_s(full_align, -1); let add = fx.bcx.ins().iadd(full_size, addend); let neg = fx.bcx.ins().ineg(full_align); let full_size = fx.bcx.ins().band(add, neg); diff --git a/src/value_and_place.rs b/src/value_and_place.rs index ae4bcad5c8ebb..c89ba0e3de9fe 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -45,7 +45,7 @@ fn codegen_field<'tcx>( // Bump the unaligned offset up to the appropriate alignment let one = fx.bcx.ins().iconst(fx.pointer_type, 1); let align_sub_1 = fx.bcx.ins().isub(unsized_align, one); - let and_lhs = fx.bcx.ins().iadd_imm(align_sub_1, unaligned_offset as i64); + let and_lhs = fx.bcx.ins().iadd_imm_u(align_sub_1, unaligned_offset as i64); let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); let and_rhs = fx.bcx.ins().isub(zero, unsized_align); let offset = fx.bcx.ins().band(and_lhs, and_rhs); @@ -268,7 +268,8 @@ impl<'tcx> CValue<'tcx> { CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(), CValueInner::ByRef(ptr, None) => { let lane_idx = clif_intcast(fx, lane_idx, fx.pointer_type, false); - let field_offset = fx.bcx.ins().imul_imm(lane_idx, lane_layout.size.bytes() as i64); + let field_offset = + fx.bcx.ins().imul_imm_u(lane_idx, lane_layout.size.bytes() as i64); let field_ptr = ptr.offset_value(fx, field_offset); CValue::by_ref(field_ptr, lane_layout) } @@ -775,7 +776,7 @@ impl<'tcx> CPlace<'tcx> { let field_offset = fx .bcx .ins() - .imul_imm(lane_idx, i64::try_from(lane_layout.size.bytes()).unwrap()); + .imul_imm_u(lane_idx, i64::try_from(lane_layout.size.bytes()).unwrap()); let field_ptr = ptr.offset_value(fx, field_offset); CPlace::for_ptr(field_ptr, lane_layout).write_cvalue(fx, value); } @@ -802,7 +803,7 @@ impl<'tcx> CPlace<'tcx> { _ => bug!("place_index({:?})", self.layout().ty), }; - let offset = fx.bcx.ins().imul_imm(index, elem_layout.size.bytes() as i64); + let offset = fx.bcx.ins().imul_imm_u(index, elem_layout.size.bytes() as i64); CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout) } From abf20f6ba191265077ac96353655b724b78deda6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:46:39 +0200 Subject: [PATCH 49/51] Use JITModule::get_address in relocate_for_jit This way we will use the actual personality function defined by the standard library version of the jitted program rather than the copy of rustc. In addition this is necessary to be able to mangle the symbol name of the personality function in the future as extern {} wouldn't be able to refer to it if the symbol name is mangled. --- src/debuginfo/emit.rs | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/debuginfo/emit.rs b/src/debuginfo/emit.rs index cc1efef287517..14235689a5576 100644 --- a/src/debuginfo/emit.rs +++ b/src/debuginfo/emit.rs @@ -81,34 +81,16 @@ impl WriterRelocate { /// Perform the collected relocations to be usable for JIT usage. #[cfg(all(feature = "jit", not(windows)))] pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec { - use cranelift_module::Module; - for reloc in self.relocs.drain(..) { match reloc.name { super::DebugRelocName::Section(_) => unreachable!(), super::DebugRelocName::Symbol(sym) => { let addr = if sym & 1 << 31 == 0 { let func_id = FuncId::from_u32(sym.try_into().unwrap()); - // FIXME make JITModule::get_address public and use it here instead. - // HACK rust_eh_personality is likely not defined in the same crate, - // so get_finalized_function won't work. Use the rust_eh_personality - // of cg_clif itself, which is likely ABI compatible. - if jit_module.declarations().get_function_decl(func_id).name.as_deref() - == Some("rust_eh_personality") - { - unsafe extern "C" { - fn rust_eh_personality() -> !; - } - rust_eh_personality as *const u8 - } else { - jit_module.get_finalized_function(func_id) - } + jit_module.get_address(&func_id.into()) } else { - jit_module - .get_finalized_data(DataId::from_u32( - u32::try_from(sym).unwrap() & !(1 << 31), - )) - .0 + let data_id = DataId::from_u32(u32::try_from(sym).unwrap() & !(1 << 31)); + jit_module.get_address(&data_id.into()) }; let val = (addr as u64 as i64 + reloc.addend) as u64; From 6da84ff8145aac19eea4807f7b669f6ca6e71a35 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:00:24 +0200 Subject: [PATCH 50/51] Rustup to rustc 1.99.0-nightly (0e29c21d9 2026-07-21) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7674a2e26ecd0..29d009597901f 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-07-19" +channel = "nightly-2026-07-22" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" From cb89537d435ec9cc9c812cd47c6130bd96aa0050 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:19:33 +0200 Subject: [PATCH 51/51] Fix rustc test suite --- scripts/test_rustc_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index b83f5f8e4e8b3..9a7769497b169 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -145,6 +145,7 @@ rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift rm -r tests/run-make/short-ice # ICE backtrace begin/end marker mismatch rm tests/ui/codegen/huge-stacks.rs # Cranelift doesn't allow stack frames to exceed 4GB rm -r tests/run-make/rustdoc/doctest/test_harness # different thread names likely caused by -Zpanic-abort-tests +rm -r tests/run-make/requires-consistent-cpu-no-native # no -Ctarget-cpu=help support # doesn't work due to the way the rustc test suite is invoked. # should work when using ./x.py test the way it is intended