From 2856338bf63c78961fa0853ffd0a2647bd4d559d Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 26 Jun 2026 17:33:04 -0400 Subject: [PATCH 1/9] Add target_feature_available_at_call_site intrinsic --- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 13 ++- compiler/rustc_codegen_llvm/src/context.rs | 4 +- compiler/rustc_codegen_llvm/src/intrinsic.rs | 38 +++++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 6 + compiler/rustc_codegen_llvm/src/llvm/mod.rs | 8 ++ compiler/rustc_codegen_ssa/src/mir/block.rs | 6 +- .../rustc_codegen_ssa/src/mir/intrinsic.rs | 108 +++++++++++++++++- .../rustc_codegen_ssa/src/target_features.rs | 2 +- .../rustc_codegen_ssa/src/traits/intrinsic.rs | 8 +- .../rustc_hir_analysis/src/check/intrinsic.rs | 7 ++ .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 68 +++++++++++ .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 20 ++++ compiler/rustc_span/src/symbol.rs | 1 + library/core/src/intrinsics/mod.rs | 27 +++++ ...rget-feature-available-at-call-site-fma.rs | 67 +++++++++++ ...arget-feature-available-at-call-site-o0.rs | 42 +++++++ ...vailable-at-call-site-rust-only-feature.rs | 33 ++++++ .../target-feature-available-at-call-site.rs | 32 ++++++ ...et-feature-available-at-call-site-const.rs | 8 ++ ...eature-available-at-call-site-const.stderr | 8 ++ ...feature-available-at-call-site-nonconst.rs | 15 +++ ...ure-available-at-call-site-nonconst.stderr | 8 ++ ...-feature-available-at-call-site-unknown.rs | 14 +++ ...ture-available-at-call-site-unknown.stderr | 8 ++ 24 files changed, 540 insertions(+), 11 deletions(-) create mode 100644 tests/assembly-llvm/target-feature-available-at-call-site-fma.rs create mode 100644 tests/codegen-llvm/target-feature-available-at-call-site-o0.rs create mode 100644 tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs create mode 100644 tests/codegen-llvm/target-feature-available-at-call-site.rs create mode 100644 tests/ui/intrinsics/target-feature-available-at-call-site-const.rs create mode 100644 tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr create mode 100644 tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs create mode 100644 tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr create mode 100644 tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs create mode 100644 tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 78a4c7e88c895..2acddf04a2ef1 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -24,7 +24,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; -use rustc_middle::{bug, span_bug}; +use rustc_middle::{bug, mir, span_bug}; use rustc_span::{Span, Symbol, sym}; use rustc_target::callconv::{ArgAbi, PassMode}; @@ -195,6 +195,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &mut self, instance: Instance<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], + _: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>>, span: Span, @@ -594,6 +595,16 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc IntrinsicResult::WroteIntoPlace } + fn codegen_target_feature_available_at_call_site( + &mut self, + _rust_feature_name: &str, + ) -> RValue<'gcc> { + // SSA already handles the easy case where the caller function has the feature enabled. + // GCC doesn't (yet) have the equiavelent LLVM pass to replace a marker post-inlining, + // so report that the feature is unavailable at the call site. + self.const_bool(false) + } + fn codegen_llvm_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 621f2cd3f9fc7..cf9dd1a6bd52c 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -36,7 +36,7 @@ use crate::back::write::to_llvm_code_model; use crate::builder::gpu_offload::{OffloadGlobals, OffloadKernelGlobals}; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; -use crate::llvm::{self, Metadata, MetadataKindId, Module, Type, Value}; +use crate::llvm::{self, Metadata, MetadataKindId, Module, TargetMachine, Type, Value}; use crate::{attributes, common, coverageinfo, debuginfo, llvm_util}; /// `TyCtxt` (and related cache datastructures) can't be move between threads. @@ -92,6 +92,7 @@ pub(crate) type CodegenCx<'ll, 'tcx> = GenericCx<'ll, FullCx<'ll, 'tcx>>; pub(crate) struct FullCx<'ll, 'tcx> { pub tcx: TyCtxt<'tcx>, pub scx: SimpleCx<'ll>, + pub tm: &'ll TargetMachine, pub use_dll_storage_attrs: bool, pub tls_model: llvm::ThreadLocalMode, @@ -660,6 +661,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { FullCx { tcx, scx: SimpleCx::new(llmod, llcx, tcx.data_layout.pointer_size()), + tm: llvm_module.tm.raw(), use_dll_storage_attrs, tls_model, codegen_unit, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 7bd604bdbbd76..6917d8a3e40e7 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -17,7 +17,7 @@ use rustc_codegen_ssa::traits::*; use rustc_hir as hir; use rustc_hir::def_id::LOCAL_CRATE; use rustc_hir::find_attr; -use rustc_middle::mir::BinOp; +use rustc_middle::mir::{self, BinOp}; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; use rustc_middle::ty::offload_meta::OffloadMetadata; use rustc_middle::ty::{self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt, TypingEnv}; @@ -32,6 +32,7 @@ use rustc_target::spec::{Arch, LlvmAbi}; use tracing::debug; use crate::abi::FnAbiLlvmExt; +use crate::attributes; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::builder::gpu_offload::{ @@ -177,6 +178,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &mut self, instance: ty::Instance<'tcx>, args: &[OperandRef<'tcx, &'ll Value>], + _: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>, span: Span, @@ -923,6 +925,40 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } + fn codegen_target_feature_available_at_call_site( + &mut self, + rust_feature_name: &str, + ) -> &'ll Value { + // SSA already returned `true` when this Rust feature is enabled on the current + // function. If there is no LLVM mapping, we cannot do any better from here. + let Some(llvm_feature) = crate::llvm_util::to_llvm_features(self.sess(), rust_feature_name) + else { + return self.const_bool(false); + }; + + // Target features can't be lost by inlining, so early return if it's already present. + // SSA should have already done this, but this is most accurate for implied features. + let mut llvm_features = llvm_feature.into_iter(); + let llvm_feature_name = llvm_features.next().unwrap().to_string(); + let enabled = llvm::FunctionHasTargetFeature(self.cx.tm, self.llfn(), &llvm_feature_name); + if enabled { + return self.const_bool(true); + } + + // If we're not optimizing, we won't run the LLVM pass + if self.sess().opts.optimize == rustc_session::config::OptLevel::No { + return self.const_bool(false); + } + + // Generate the marker to be replaced by the LLVM pass + let marker_name = format!("rust.target_feature_available_at_call_site.{llvm_feature_name}"); + let fn_ty = self.type_func(&[], self.type_i1()); + let llfn = self.cx.declare_cfn(&marker_name, llvm::UnnamedAddr::No, fn_ty); + let nounwind = llvm::AttributeKind::NoUnwind.create_attr(self.cx.llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[nounwind]); + self.call(fn_ty, None, None, llfn, &[], None, None) + } + fn codegen_llvm_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 56c90582e4642..673f60d3be340 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2406,6 +2406,12 @@ unsafe extern "C" { pub(crate) fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; pub(crate) fn LLVMRustTargetHasMnemonic(T: &TargetMachine, s: *const c_char) -> bool; + pub(crate) fn LLVMRustFunctionHasTargetFeature( + TM: &TargetMachine, + F: &Value, + Feature: *const c_char, + FeatureLen: size_t, + ) -> bool; pub(crate) fn LLVMRustPrintTargetCPUs(TM: &TargetMachine, OutStr: &RustString); pub(crate) fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index e9bc6ae0e80ef..c5e4e11940a63 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -51,6 +51,14 @@ pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: &str) { unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } } +pub(crate) fn FunctionHasTargetFeature<'ll>( + tm: &'ll TargetMachine, + llfn: &'ll Value, + feature: &str, +) -> bool { + unsafe { LLVMRustFunctionHasTargetFeature(tm, llfn, feature.as_c_char_ptr(), feature.len()) } +} + pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 0173b84a4d9a1..29a613dff1241 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1008,13 +1008,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { span_bug!(self.mir.span, "can't directly store to unaligned value"); } - let args: Vec<_> = + let mir_args: Vec<_> = args.iter().map(|arg| arg.node.clone()).collect(); + let codegen_args: Vec<_> = args.iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect(); let intrinsic_result = self.codegen_intrinsic_call( bx, instance, - &args, + &codegen_args, + &mir_args, result_layout, result_place, source_info, diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 99546bea65959..638c224c29971 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,9 +1,12 @@ +use itertools::Itertools as _; use rustc_abi::{Align, FieldIdx, WrappingRange}; -use rustc_middle::mir::SourceInfo; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::mir::{self, SourceInfo}; +use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; -use rustc_span::{ErrorGuaranteed, sym}; +use rustc_span::{ErrorGuaranteed, Symbol, sym}; use rustc_target::spec::Arch; use super::operand::{OperandRef, OperandValue}; @@ -59,6 +62,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &mut Bx, instance: ty::Instance<'tcx>, args: &[OperandRef<'tcx, Bx::Value>], + mir_args: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>, source_info: SourceInfo, @@ -589,10 +593,85 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::ZeroSized } + sym::target_feature_available_at_call_site => { + // This intrinsic either returns a bool reflecting feature presence, or a marker. A later + // LLVM pass will replace the marker with the feature presence. + // * If the feature is already known to be present, we can immediately return true. + // * If we can detect the feature later in the LLVM pass, return the marker. + // * If the feature isn't present and we can't detect it later, return false. + + // Return false on errors to allow diagnostics to continue + fn err_false<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + span: rustc_span::Span, + msg: impl Into, + ) -> IntrinsicResult<'tcx, Bx::Value> { + bx.tcx().dcx().span_err(span, msg); + IntrinsicResult::Operand(OperandValue::Immediate(bx.const_bool(false))) + } + + let [feature_arg] = mir_args else { + span_bug!(span, "wrong number of MIR arguments for {name}"); + }; + + let Some(feature_name) = feature_arg + .constant() + .or_else(|| match feature_arg { + // This intrinsic takes a string literal, but in MIR those are sometimes passed via `let x = "constant"` + mir::Operand::Copy(place) | mir::Operand::Move(place) => { + constant_assigned_to_local(&self.mir, place.as_local()?) + } + _ => None, + }) + .map(|feature_const| self.eval_mir_constant(feature_const)) + .and_then(|feature_const| { + // This isn't a diagnostic, but it's similar usage as we aren't using it within the interpreter. + feature_const.try_get_slice_bytes_for_diagnostics(self.cx.tcx()) + }) + .and_then(|feature_bytes| std::str::from_utf8(feature_bytes).ok()) + else { + return err_false( + bx, + span, + "`target_feature_available_at_call_site` requires a string literal argument", + ); + }; + + // Ensure it's a valid Rust feature + let rust_target_features = bx.tcx().rust_target_features(LOCAL_CRATE); + let Some(&stability) = rust_target_features.get(feature_name) else { + return err_false(bx, span, format!("unknown target feature `{feature_name}`")); + }; + if let Err(reason) = stability.toggle_allowed() { + return err_false( + bx, + span, + format!("cannot use target feature `{feature_name}`: {reason}"), + ); + } + + // If the function has the feature, we can emit true now. Otherwise emit the marker. + let current_fn_features = + crate::target_features::asm_target_features(bx.tcx(), self.instance.def_id()); + if current_fn_features.contains(&Symbol::intern(feature_name)) { + OperandValue::Immediate(bx.const_bool(true)) + } else { + OperandValue::Immediate( + bx.codegen_target_feature_available_at_call_site(feature_name), + ) + } + } + _ => { // Need to use backend-specific things in the implementation. - let result = - bx.codegen_intrinsic_call(instance, args, result_layout, result_place, span); + let result = bx.codegen_intrinsic_call( + instance, + args, + mir_args, + result_layout, + result_place, + span, + ); if let IntrinsicResult::Operand(op) = result { op } else { @@ -634,3 +713,24 @@ fn float_type_width(ty: Ty<'_>) -> Option { _ => None, } } + +// Reads a constant through one level of assignment, e.g. `let x = "constant"` +fn constant_assigned_to_local<'mir, 'tcx>( + mir: &'mir mir::Body<'tcx>, + local: mir::Local, +) -> Option<&'mir mir::ConstOperand<'tcx>> { + let (_, rvalue) = mir + .basic_blocks + .iter() + .flat_map(|bb| bb.statements.iter()) + .filter_map(|stmt| stmt.kind.as_assign()) + .filter(|(place, _)| place.as_local() == Some(local)) + .exactly_one() + .ok()?; + + let mir::Rvalue::Use(operand, _) = rvalue else { + return None; + }; + + operand.constant() +} diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index 3c771f0eb7ec4..d02c6e43a85a5 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -130,7 +130,7 @@ pub(crate) fn from_target_feature_attr( /// Computes the set of target features used in a function for the purposes of /// inline assembly. -fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxIndexSet { +pub(crate) fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxIndexSet { let mut target_features = tcx.sess.unstable_target_features.clone(); if tcx.def_kind(did).has_codegen_attrs() { let attrs = tcx.codegen_fn_attrs(did); diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index 47144834b5072..7e4b17b241368 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -1,4 +1,4 @@ -use rustc_middle::ty; +use rustc_middle::{mir, ty}; use rustc_span::Span; use super::BackendTypes; @@ -26,11 +26,17 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { &mut self, instance: ty::Instance<'tcx>, args: &[OperandRef<'tcx, Self::Value>], + mir_args: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>, span: Span, ) -> IntrinsicResult<'tcx, Self::Value>; + fn codegen_target_feature_available_at_call_site( + &mut self, + llvm_feature_name: &str, + ) -> Self::Value; + fn codegen_llvm_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 9dd7bb8058afc..6d907de51798b 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -207,6 +207,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::sqrtf64 | sym::sqrtf128 | sym::sub_with_overflow + | sym::target_feature_available_at_call_site | sym::three_way_compare | sym::truncf16 | sym::truncf32 @@ -683,6 +684,12 @@ pub(crate) fn check_intrinsic_type( sym::black_box => (1, 0, vec![param(0)], param(0)), sym::is_val_statically_known => (1, 0, vec![param(0)], tcx.types.bool), + sym::target_feature_available_at_call_site => { + let br = ty::BoundRegion { var: ty::BoundVar::ZERO, kind: ty::BoundRegionKind::Anon }; + let feature_name = + Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), tcx.types.str_); + (0, 0, vec![feature_name], tcx.types.bool) + } sym::const_eval_select => (4, 0, vec![param(0), param(1), param(2)], param(3)), diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index fe5b8edce4a1d..d0be3cc321c8f 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -12,6 +12,7 @@ #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/AssemblyAnnotationWriter.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/LegacyPassManager.h" @@ -49,6 +50,8 @@ #include "llvm/Transforms/Instrumentation/RealtimeSanitizer.h" #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/Scalar/AnnotationRemarks.h" +#include "llvm/Transforms/Scalar/InstSimplifyPass.h" +#include "llvm/Transforms/Scalar/SimplifyCFG.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" @@ -118,6 +121,65 @@ extern "C" bool LLVMRustTargetHasMnemonic(LLVMTargetMachineRef TM, return false; } +static constexpr StringLiteral TargetFeatureAvailableAtCallSitePrefix( + "rust.target_feature_available_at_call_site."); + +class TargetFeatureAvailableAtCallSitePass + : public PassInfoMixin { + TargetMachine *TM; + +public: + explicit TargetFeatureAvailableAtCallSitePass(TargetMachine *TM) : TM(TM) {} + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM) { + SmallVector CallsToErase; + DenseSet ChangedFunctions; + for (Function &MarkerDecl : M.functions()) { + if (!MarkerDecl.getName().starts_with( + TargetFeatureAvailableAtCallSitePrefix)) + continue; + + StringRef Feature = MarkerDecl.getName().drop_front( + TargetFeatureAvailableAtCallSitePrefix.size()); + SmallString<64> EnabledFeature("+"); + EnabledFeature += Feature; + + for (User *U : MarkerDecl.users()) { + auto *Call = dyn_cast(U); + if (!Call || Call->getCalledFunction() != &MarkerDecl) + continue; + + Function *Caller = Call->getFunction(); + const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(*Caller); + bool Enabled = + Subtarget != nullptr && Subtarget->checkFeatures(EnabledFeature); + + Call->replaceAllUsesWith(ConstantInt::getBool(M.getContext(), Enabled)); + CallsToErase.push_back(Call); + ChangedFunctions.insert(Caller); + } + } + + if (CallsToErase.empty()) + return PreservedAnalyses::all(); + + for (CallInst *Call : CallsToErase) + Call->eraseFromParent(); + + auto &FAM = + MAM.getResult(M).getManager(); + FunctionPassManager FPM; + FPM.addPass(InstSimplifyPass()); + FPM.addPass(SimplifyCFGPass()); + for (Function *F : ChangedFunctions) + FPM.run(*F, FAM); + + PreservedAnalyses PA = PreservedAnalyses::none(); + PA.preserve(); + return PA; + } +}; + enum class LLVMRustCodeModel { Tiny, Small, @@ -727,6 +789,12 @@ extern "C" LLVMRustResult LLVMRustOptimize( ThinOrFullLTOPhase)>> OptimizerLastEPCallbacks; + OptimizerLastEPCallbacks.push_back([TM](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { + MPM.addPass(TargetFeatureAvailableAtCallSitePass(TM)); + }); + if (!IsLinkerPluginLTO && SanitizerOptions && SanitizerOptions->SanitizeCFI && !NoPrepopulatePasses) { PipelineStartEPCallbacks.push_back( diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 3b8e6f6415365..50c50f7e39499 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -9,6 +9,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -36,6 +37,7 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/Timer.h" #include "llvm/Support/ToolOutputFile.h" +#include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ValueMapper.h" #include @@ -68,6 +70,9 @@ using namespace llvm; using namespace llvm::sys; using namespace llvm::object; +typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; +DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + // This opcode is an LLVM detail that could hypothetically change (?), so // verify that the hard-coded value in `dwarf_const.rs` still agrees with LLVM. static_assert(dwarf::DW_OP_LLVM_fragment == 0x1000); @@ -1096,6 +1101,21 @@ extern "C" void LLVMRustRemoveFnAttribute(LLVMValueRef Fn, const char *Name, } } +extern "C" bool LLVMRustFunctionHasTargetFeature(LLVMTargetMachineRef TMRef, + LLVMValueRef F, + const char *Feature, + size_t FeatureLen) { + if (auto *Fn = dyn_cast(unwrap(F))) { + TargetMachine *TM = unwrap(TMRef); + if (const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(*Fn)) { + SmallString<64> EnabledFeature("+"); + EnabledFeature += StringRef(Feature, FeatureLen); + return Subtarget->checkFeatures(EnabledFeature); + } + } + return false; +} + extern "C" void LLVMRustGlobalAddMetadata(LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { unwrap(Global)->addMetadata(Kind, *unwrap(MD)); diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 71eae246ebaff..83f445923626c 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2076,6 +2076,7 @@ symbols! { target_family, target_feature, target_feature_11, + target_feature_available_at_call_site, target_feature_inline_always, target_has_atomic, target_has_atomic_load_store, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index bc96c768c0c94..ca054fece05a9 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3679,3 +3679,30 @@ pub const unsafe fn va_end(ap: &mut VaList<'_>) { pub fn return_address() -> *const () { core::ptr::null() } + +/// Returns whether the named target feature is available at the current call site. +/// +/// "Available" in this context means known to be enabled in the calling function. +/// At a minimum, being enabled globally or in `#[target_features]` of the function containing +/// this intrinsic is considered available. Depending on codegen options and backend, the features +/// of the function the intrinsic call is inlined into may also be accounted for. This is intended +/// to support negligible or zero overhead branching on feature availability, for scenarios where +/// runtime detection has too much overhead and globally enabled features are not sufficient. +/// +/// Callers should pass a string literal naming a Rust target feature, such as `"avx"` or `"neon"`. +/// The compiler may accept a few additional simple constant forms, but this intrinsic should be +/// treated as taking a string literal. +/// +/// The LLVM backend implements this intrinsic by lowering to a marker that is resolved by a +/// post-inlining pass. +/// +/// # Safety +/// +/// Code relying on the result of this intrinsic must be sound for both `true` and `false`. +/// Even calling the same function multiple times can result in different return values, +/// depending on how the function was inlined. In some cases, this intrinsic might still return +/// `false` for features that are enabled in the inlined function. +#[rustc_nounwind] +#[unstable(feature = "core_intrinsics", issue = "none")] +#[rustc_intrinsic] +pub fn target_feature_available_at_call_site(feature: &str) -> bool; diff --git a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs new file mode 100644 index 0000000000000..b8e307882cfe1 --- /dev/null +++ b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs @@ -0,0 +1,67 @@ +//@ assembly-output: emit-asm +//@ compile-flags: --crate-type=lib -Copt-level=3 -C llvm-args=-x86-asm-syntax=intel --target=x86_64-unknown-linux-gnu -Ctarget-feature=-avx,-fma +//@ needs-llvm-components: x86 +//@ only-x86_64 +//@ ignore-sgx +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::arch::x86_64::{_mm_add_ss, _mm_cvtss_f32, _mm_fmadd_ss, _mm_mul_ss, _mm_set_ss}; +use std::intrinsics::target_feature_available_at_call_site; + +#[inline(always)] +fn maybe_fma(x: f32, y: f32, z: f32) -> f32 { + if target_feature_available_at_call_site("fma") { x.mul_add(y, z) } else { x * y + z } +} + +#[inline(always)] +fn maybe_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { + unsafe { + let x = _mm_set_ss(x); + let y = _mm_set_ss(y); + let z = _mm_set_ss(z); + let result = if target_feature_available_at_call_site("fma") { + _mm_fmadd_ss(x, y, z) + } else { + _mm_add_ss(_mm_mul_ss(x, y), z) + }; + _mm_cvtss_f32(result) + } +} + +// CHECK-LABEL: with_fma: +// CHECK: vfmadd +// CHECK-NOT: mulss +// CHECK-NOT: addss +#[no_mangle] +#[target_feature(enable = "avx,fma")] +pub fn with_fma(x: f32, y: f32, z: f32) -> f32 { + maybe_fma(x, y, z) +} + +// CHECK-LABEL: without_fma: +// CHECK: mulss +// CHECK: addss +// CHECK-NOT: vfmadd +#[no_mangle] +pub fn without_fma(x: f32, y: f32, z: f32) -> f32 { + maybe_fma(x, y, z) +} + +#[no_mangle] +#[target_feature(enable = "avx,fma")] +pub fn with_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { + maybe_fma_arch_intrinsic(x, y, z) +} + +// CHECK-LABEL: without_fma_arch_intrinsic: +// CHECK: mulss +// CHECK: addss +// CHECK-NOT: vfmadd +#[no_mangle] +pub fn without_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { + maybe_fma_arch_intrinsic(x, y, z) +} + +// CHECK: with_fma_arch_intrinsic = with_fma diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs new file mode 100644 index 0000000000000..cbb5936011365 --- /dev/null +++ b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs @@ -0,0 +1,42 @@ +//@ revisions: NEG_ONLY NEG_POS POS_NEG +//@ compile-flags: --crate-type=lib -Copt-level=0 --target=x86_64-unknown-linux-gnu +//@ [NEG_ONLY] compile-flags: -Ctarget-feature=-fma +//@ [NEG_POS] compile-flags: -Ctarget-feature=-fma,+fma +//@ [POS_NEG] compile-flags: -Ctarget-feature=+fma,-fma +//@ needs-llvm-components: x86 +//@ ignore-backends: gcc + +// opt-level=0 has different inlining properties, so ensure that the pass still works. + +#![feature(core_intrinsics)] + +use std::intrinsics::target_feature_available_at_call_site; + +// NEG_ONLY-LABEL: @with_fma( +// NEG_ONLY-NOT: rust.target_feature_available_at_call_site +// NEG_ONLY: ret i1 true +// NEG_POS-LABEL: @with_fma( +// NEG_POS-NOT: rust.target_feature_available_at_call_site +// NEG_POS: ret i1 true +// POS_NEG-LABEL: @with_fma( +// POS_NEG-NOT: rust.target_feature_available_at_call_site +// POS_NEG: ret i1 true +#[no_mangle] +#[target_feature(enable = "fma")] +pub fn with_fma() -> bool { + target_feature_available_at_call_site("fma") +} + +// NEG_ONLY-LABEL: @without_fma( +// NEG_ONLY-NOT: rust.target_feature_available_at_call_site +// NEG_ONLY: ret i1 false +// NEG_POS-LABEL: @without_fma( +// NEG_POS-NOT: rust.target_feature_available_at_call_site +// NEG_POS: ret i1 true +// POS_NEG-LABEL: @without_fma( +// POS_NEG-NOT: rust.target_feature_available_at_call_site +// POS_NEG: ret i1 false +#[no_mangle] +pub fn without_fma() -> bool { + target_feature_available_at_call_site("fma") +} diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs new file mode 100644 index 0000000000000..a55bf028d4eca --- /dev/null +++ b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs @@ -0,0 +1,33 @@ +//@ add-minicore +//@ compile-flags: --crate-type=lib -Copt-level=3 --target=aarch64-unknown-linux-gnu +//@ needs-llvm-components: aarch64 +//@ ignore-backends: gcc + +// Test intrinsic on a feature that doesn't map to an LLVM feature. +// Can't resolve after inlining, but can get the current function's +// feature presence. + +#![feature(no_core, lang_items, intrinsics)] +#![no_core] + +extern crate minicore; + +#[rustc_intrinsic] +pub fn target_feature_available_at_call_site(feature: &str) -> bool; + +// CHECK-LABEL: @with_tme( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i1 true +#[no_mangle] +#[target_feature(enable = "tme")] +pub fn with_tme() -> bool { + target_feature_available_at_call_site("tme") +} + +// CHECK-LABEL: @without_tme( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i1 false +#[no_mangle] +pub fn without_tme() -> bool { + target_feature_available_at_call_site("tme") +} diff --git a/tests/codegen-llvm/target-feature-available-at-call-site.rs b/tests/codegen-llvm/target-feature-available-at-call-site.rs new file mode 100644 index 0000000000000..e1afbd1f5b4bd --- /dev/null +++ b/tests/codegen-llvm/target-feature-available-at-call-site.rs @@ -0,0 +1,32 @@ +//@ compile-flags: --crate-type=lib -Copt-level=3 --target=x86_64-unknown-linux-gnu -Ctarget-feature=-avx +//@ needs-llvm-components: x86 +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::intrinsics::target_feature_available_at_call_site; + +#[inline] +pub fn avx_branch_value() -> i32 { + if target_feature_available_at_call_site("avx") { 1 } else { 0 } +} + +// CHECK-LABEL: @with_avx( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i32 1 +#[no_mangle] +#[target_feature(enable = "avx")] +pub fn with_avx() -> i32 { + avx_branch_value() +} + +// CHECK-LABEL: @without_avx( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i32 0 +#[no_mangle] +pub fn without_avx() -> i32 { + avx_branch_value() +} + +// CHECK: attributes #0 = {{.*}}"target-features"="{{[^"]*}}+avx{{.*}}" +// CHECK: attributes #1 = {{.*}}"target-features"="{{[^"]*}}-avx{{.*}}" diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs new file mode 100644 index 0000000000000..d9c177e68ea19 --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs @@ -0,0 +1,8 @@ +#![feature(core_intrinsics)] + +const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site("fma"); +//~^ ERROR cannot call non-const intrinsic `target_feature_available_at_call_site` in constants + +fn main() { + let _ = HAS_FMA; +} diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr new file mode 100644 index 0000000000000..9f843454251f6 --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr @@ -0,0 +1,8 @@ +error: cannot call non-const intrinsic `target_feature_available_at_call_site` in constants + --> $DIR/target-feature-available-at-call-site-const.rs:3:23 + | +LL | const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site("fma"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs new file mode 100644 index 0000000000000..3b4c19f3b1e56 --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs @@ -0,0 +1,15 @@ +//@ compile-flags: --target=x86_64-unknown-linux-gnu --crate-type=lib --emit=llvm-ir +//@ needs-llvm-components: x86 +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::hint::black_box; +use std::intrinsics::target_feature_available_at_call_site; + +#[no_mangle] +pub fn check() { + let feature = black_box("fma"); + let _ = target_feature_available_at_call_site(feature); + //~^ ERROR `target_feature_available_at_call_site` requires a string literal argument +} diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr new file mode 100644 index 0000000000000..5e036d21c3cee --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr @@ -0,0 +1,8 @@ +error: `target_feature_available_at_call_site` requires a string literal argument + --> $DIR/target-feature-available-at-call-site-nonconst.rs:13:13 + | +LL | let _ = target_feature_available_at_call_site(feature); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs new file mode 100644 index 0000000000000..f577ef33d63d3 --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs @@ -0,0 +1,14 @@ +//@ compile-flags: --target=x86_64-unknown-linux-gnu --crate-type=lib --emit=llvm-ir +//@ needs-llvm-components: x86 +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::hint::black_box; +use std::intrinsics::target_feature_available_at_call_site; + +#[no_mangle] +pub fn check() { + let _ = target_feature_available_at_call_site("definitely-not-a-feature"); + //~^ ERROR unknown target feature `definitely-not-a-feature` +} diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr new file mode 100644 index 0000000000000..46ddf1185d139 --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr @@ -0,0 +1,8 @@ +error: unknown target feature `definitely-not-a-feature` + --> $DIR/target-feature-available-at-call-site-unknown.rs:12:13 + | +LL | let _ = target_feature_available_at_call_site("definitely-not-a-feature"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + From eceab3557b77098713fcee9ea2955a876563a9f5 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 3 Jul 2026 12:05:43 -0400 Subject: [PATCH 2/9] Improve documentation --- compiler/rustc_codegen_gcc/src/intrinsic/mod.rs | 2 +- compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 7 +++++++ library/core/src/intrinsics/mod.rs | 14 ++++++++++---- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 2acddf04a2ef1..fab6fa836b4d8 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -600,7 +600,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc _rust_feature_name: &str, ) -> RValue<'gcc> { // SSA already handles the easy case where the caller function has the feature enabled. - // GCC doesn't (yet) have the equiavelent LLVM pass to replace a marker post-inlining, + // GCC doesn't (yet) have the equivalent LLVM pass to replace a marker post-inlining, // so report that the feature is unavailable at the call site. self.const_bool(false) } diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index d0be3cc321c8f..08dbdd15d6752 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -124,6 +124,13 @@ extern "C" bool LLVMRustTargetHasMnemonic(LLVMTargetMachineRef TM, static constexpr StringLiteral TargetFeatureAvailableAtCallSitePrefix( "rust.target_feature_available_at_call_site."); +/// During MIR lowering, the target_feature_available_at_call_site intrinsic +/// is lowered to a marker function call, for example: +/// rust.target_feature_available_at_call_site.avx +/// +/// This pass replaces those markers with true or false depending on whether the +/// feature is enabled in the caller. To be useful, this pass must run after +/// inlining. class TargetFeatureAvailableAtCallSitePass : public PassInfoMixin { TargetMachine *TM; diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index ca054fece05a9..3bd0139fb565c 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3681,6 +3681,8 @@ pub fn return_address() -> *const () { } /// Returns whether the named target feature is available at the current call site. +/// This will not emit any runtime detection code; it always turns into `true` or `false` at some +/// point during compilation. /// /// "Available" in this context means known to be enabled in the calling function. /// At a minimum, being enabled globally or in `#[target_features]` of the function containing @@ -3698,10 +3700,14 @@ pub fn return_address() -> *const () { /// /// # Safety /// -/// Code relying on the result of this intrinsic must be sound for both `true` and `false`. -/// Even calling the same function multiple times can result in different return values, -/// depending on how the function was inlined. In some cases, this intrinsic might still return -/// `false` for features that are enabled in the inlined function. +/// The return value of this intrinsic is call-site dependent and may vary between calls to the +/// same function, depending on inlining and codegen. Callers must not rely on the result being +/// stable across calls or multiple uses of the same function containing the intrinsic. +/// +/// A return value of `true` guarantees that the feature is enabled at the call site and it is +/// safe to call other functions that require the feature. +/// A return value of `false` carries no information and does not indicate whether the feature is +/// enabled or not. #[rustc_nounwind] #[unstable(feature = "core_intrinsics", issue = "none")] #[rustc_intrinsic] From b4c7a51108f5796ddd31f753070e707751a5ea94 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 3 Jul 2026 14:34:08 -0400 Subject: [PATCH 3/9] Pass feature as metadata argument and move pass to OptimizerEarly --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 13 ++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + compiler/rustc_codegen_llvm/src/type_.rs | 4 + .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 80 +++++++++---------- ...rget-feature-available-at-call-site-fma.rs | 5 +- 5 files changed, 53 insertions(+), 50 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 6917d8a3e40e7..4a6702055f9f5 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -951,12 +951,17 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } // Generate the marker to be replaced by the LLVM pass - let marker_name = format!("rust.target_feature_available_at_call_site.{llvm_feature_name}"); - let fn_ty = self.type_func(&[], self.type_i1()); - let llfn = self.cx.declare_cfn(&marker_name, llvm::UnnamedAddr::No, fn_ty); + let fn_ty = self.type_func(&[self.cx.type_metadata()], self.type_i1()); + let llfn = self.cx.declare_cfn( + "rust.target_feature_available_at_call_site", + llvm::UnnamedAddr::No, + fn_ty, + ); let nounwind = llvm::AttributeKind::NoUnwind.create_attr(self.cx.llcx); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[nounwind]); - self.call(fn_ty, None, None, llfn, &[], None, None) + let feature_arg = + self.cx.get_metadata_value(self.cx.create_metadata(llvm_feature_name.as_bytes())); + self.call(fn_ty, None, None, llfn, &[feature_arg], None, None) } fn codegen_llvm_intrinsic_call( diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 673f60d3be340..6285eb23a6ff4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -944,6 +944,7 @@ unsafe extern "C" { pub(crate) fn LLVMFloatTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on non-IEEE real types pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 0d49971f52533..8001b917026e9 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -59,6 +59,10 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { unsafe { llvm::LLVMVoidTypeInContext(self.llcx()) } } + pub(crate) fn type_metadata(&self) -> &'ll Type { + unsafe { llvm::LLVMMetadataTypeInContext(self.llcx()) } + } + ///x Creates an integer type with the given number of bits, e.g., i24 pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { llvm::LLVMIntTypeInContext(self.llcx(), num_bits as c_uint) diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 08dbdd15d6752..8e8d14cdb658d 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -16,6 +16,7 @@ #include "llvm/IR/AssemblyAnnotationWriter.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" #include "llvm/IRPrinter/IRPrintingPasses.h" @@ -50,8 +51,6 @@ #include "llvm/Transforms/Instrumentation/RealtimeSanitizer.h" #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/Scalar/AnnotationRemarks.h" -#include "llvm/Transforms/Scalar/InstSimplifyPass.h" -#include "llvm/Transforms/Scalar/SimplifyCFG.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" @@ -121,12 +120,9 @@ extern "C" bool LLVMRustTargetHasMnemonic(LLVMTargetMachineRef TM, return false; } -static constexpr StringLiteral TargetFeatureAvailableAtCallSitePrefix( - "rust.target_feature_available_at_call_site."); - /// During MIR lowering, the target_feature_available_at_call_site intrinsic -/// is lowered to a marker function call, for example: -/// rust.target_feature_available_at_call_site.avx +/// is lowered to a marker function call carrying the LLVM feature as a +/// metadata string argument. /// /// This pass replaces those markers with true or false depending on whether the /// feature is enabled in the caller. To be useful, this pass must run after @@ -138,33 +134,36 @@ class TargetFeatureAvailableAtCallSitePass public: explicit TargetFeatureAvailableAtCallSitePass(TargetMachine *TM) : TM(TM) {} - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM) { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { + Function *MarkerDecl = + M.getFunction("rust.target_feature_available_at_call_site"); + if (MarkerDecl == nullptr) + return PreservedAnalyses::all(); + SmallVector CallsToErase; - DenseSet ChangedFunctions; - for (Function &MarkerDecl : M.functions()) { - if (!MarkerDecl.getName().starts_with( - TargetFeatureAvailableAtCallSitePrefix)) + for (User *U : MarkerDecl->users()) { + auto *Call = dyn_cast(U); + if (!Call || Call->getCalledFunction() != MarkerDecl || + Call->arg_size() != 1) continue; - StringRef Feature = MarkerDecl.getName().drop_front( - TargetFeatureAvailableAtCallSitePrefix.size()); - SmallString<64> EnabledFeature("+"); - EnabledFeature += Feature; + auto *FeatureAsValue = dyn_cast(Call->getArgOperand(0)); + auto *FeatureMetadata = + FeatureAsValue ? dyn_cast(FeatureAsValue->getMetadata()) + : nullptr; + if (FeatureMetadata == nullptr) + continue; - for (User *U : MarkerDecl.users()) { - auto *Call = dyn_cast(U); - if (!Call || Call->getCalledFunction() != &MarkerDecl) - continue; + SmallString<64> EnabledFeature("+"); + EnabledFeature += FeatureMetadata->getString(); - Function *Caller = Call->getFunction(); - const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(*Caller); - bool Enabled = - Subtarget != nullptr && Subtarget->checkFeatures(EnabledFeature); + Function *Caller = Call->getFunction(); + const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(*Caller); + bool Enabled = + Subtarget != nullptr && Subtarget->checkFeatures(EnabledFeature); - Call->replaceAllUsesWith(ConstantInt::getBool(M.getContext(), Enabled)); - CallsToErase.push_back(Call); - ChangedFunctions.insert(Caller); - } + Call->replaceAllUsesWith(ConstantInt::getBool(M.getContext(), Enabled)); + CallsToErase.push_back(Call); } if (CallsToErase.empty()) @@ -173,17 +172,7 @@ class TargetFeatureAvailableAtCallSitePass for (CallInst *Call : CallsToErase) Call->eraseFromParent(); - auto &FAM = - MAM.getResult(M).getManager(); - FunctionPassManager FPM; - FPM.addPass(InstSimplifyPass()); - FPM.addPass(SimplifyCFGPass()); - for (Function *F : ChangedFunctions) - FPM.run(*F, FAM); - - PreservedAnalyses PA = PreservedAnalyses::none(); - PA.preserve(); - return PA; + return PreservedAnalyses::none(); } }; @@ -792,13 +781,16 @@ extern "C" LLVMRustResult LLVMRustOptimize( // the PassBuilder does not create a pipeline. std::vector> PipelineStartEPCallbacks; + std::vector> + OptimizerEarlyEPCallbacks; std::vector> OptimizerLastEPCallbacks; - OptimizerLastEPCallbacks.push_back([TM](ModulePassManager &MPM, - OptimizationLevel Level, - ThinOrFullLTOPhase Phase) { + OptimizerEarlyEPCallbacks.push_back([TM](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { MPM.addPass(TargetFeatureAvailableAtCallSitePass(TM)); }); @@ -939,6 +931,8 @@ extern "C" LLVMRustResult LLVMRustOptimize( if (!NoPrepopulatePasses) { for (const auto &C : PipelineStartEPCallbacks) PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerEarlyEPCallbacks) + PB.registerOptimizerEarlyEPCallback(C); for (const auto &C : OptimizerLastEPCallbacks) PB.registerOptimizerLastEPCallback(C); @@ -993,6 +987,8 @@ extern "C" LLVMRustResult LLVMRustOptimize( // add the verifier, instrumentation, etc passes if they were requested for (const auto &C : PipelineStartEPCallbacks) C(MPM, OptLevel); + for (const auto &C : OptimizerEarlyEPCallbacks) + C(MPM, OptLevel, ThinOrFullLTOPhase::None); for (const auto &C : OptimizerLastEPCallbacks) C(MPM, OptLevel, ThinOrFullLTOPhase::None); } diff --git a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs index b8e307882cfe1..e89a199bbe632 100644 --- a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs +++ b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs @@ -55,13 +55,10 @@ pub fn with_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { maybe_fma_arch_intrinsic(x, y, z) } -// CHECK-LABEL: without_fma_arch_intrinsic: -// CHECK: mulss -// CHECK: addss -// CHECK-NOT: vfmadd #[no_mangle] pub fn without_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { maybe_fma_arch_intrinsic(x, y, z) } // CHECK: with_fma_arch_intrinsic = with_fma +// CHECK: without_fma_arch_intrinsic = without_fma From 62cc384c5a804b515c46ed74dc77f37f9cd52e0f Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 3 Jul 2026 20:54:05 -0400 Subject: [PATCH 4/9] Pass feature name as a const generic --- .../rustc_codegen_ssa/src/mir/intrinsic.rs | 53 ++++--------------- .../rustc_hir_analysis/src/check/intrinsic.rs | 7 +-- library/core/src/intrinsics/mod.rs | 38 +++++++++++-- ...rget-feature-available-at-call-site-fma.rs | 7 ++- ...arget-feature-available-at-call-site-o0.rs | 8 +-- ...vailable-at-call-site-rust-only-feature.rs | 17 +++--- .../target-feature-available-at-call-site.rs | 6 +-- ...et-feature-available-at-call-site-const.rs | 2 +- ...eature-available-at-call-site-const.stderr | 6 ++- ...feature-available-at-call-site-nonconst.rs | 7 ++- ...ure-available-at-call-site-nonconst.stderr | 12 +++-- ...-feature-available-at-call-site-unknown.rs | 5 +- ...ture-available-at-call-site-unknown.stderr | 8 +-- 13 files changed, 85 insertions(+), 91 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 638c224c29971..19ffb437e1850 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,8 +1,6 @@ -use itertools::Itertools as _; use rustc_abi::{Align, FieldIdx, WrappingRange}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::mir::{self, SourceInfo}; -use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; @@ -128,6 +126,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | sym::atomic_fence | sym::atomic_singlethreadfence | sym::caller_location + | sym::target_feature_available_at_call_site | sym::return_address => {} _ => { span_bug!( @@ -610,30 +609,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { IntrinsicResult::Operand(OperandValue::Immediate(bx.const_bool(false))) } - let [feature_arg] = mir_args else { - span_bug!(span, "wrong number of MIR arguments for {name}"); + let Some(feature_bytes) = fn_args + .const_at(0) + .try_to_value() + .and_then(|feature| feature.try_to_raw_bytes(bx.tcx())) + else { + span_bug!(span, "wrong const argument for {name}"); }; - - let Some(feature_name) = feature_arg - .constant() - .or_else(|| match feature_arg { - // This intrinsic takes a string literal, but in MIR those are sometimes passed via `let x = "constant"` - mir::Operand::Copy(place) | mir::Operand::Move(place) => { - constant_assigned_to_local(&self.mir, place.as_local()?) - } - _ => None, - }) - .map(|feature_const| self.eval_mir_constant(feature_const)) - .and_then(|feature_const| { - // This isn't a diagnostic, but it's similar usage as we aren't using it within the interpreter. - feature_const.try_get_slice_bytes_for_diagnostics(self.cx.tcx()) - }) - .and_then(|feature_bytes| std::str::from_utf8(feature_bytes).ok()) + let feature_len = + feature_bytes.iter().position(|&byte| byte == 0).unwrap_or(feature_bytes.len()); + let Some(feature_name) = std::str::from_utf8(&feature_bytes[..feature_len]).ok() else { return err_false( bx, span, - "`target_feature_available_at_call_site` requires a string literal argument", + "`target_feature_available_at_call_site` requires a UTF-8 feature name", ); }; @@ -713,24 +703,3 @@ fn float_type_width(ty: Ty<'_>) -> Option { _ => None, } } - -// Reads a constant through one level of assignment, e.g. `let x = "constant"` -fn constant_assigned_to_local<'mir, 'tcx>( - mir: &'mir mir::Body<'tcx>, - local: mir::Local, -) -> Option<&'mir mir::ConstOperand<'tcx>> { - let (_, rvalue) = mir - .basic_blocks - .iter() - .flat_map(|bb| bb.statements.iter()) - .filter_map(|stmt| stmt.kind.as_assign()) - .filter(|(place, _)| place.as_local() == Some(local)) - .exactly_one() - .ok()?; - - let mir::Rvalue::Use(operand, _) = rvalue else { - return None; - }; - - operand.constant() -} diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 6d907de51798b..5baac385145ff 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -684,12 +684,7 @@ pub(crate) fn check_intrinsic_type( sym::black_box => (1, 0, vec![param(0)], param(0)), sym::is_val_statically_known => (1, 0, vec![param(0)], tcx.types.bool), - sym::target_feature_available_at_call_site => { - let br = ty::BoundRegion { var: ty::BoundVar::ZERO, kind: ty::BoundRegionKind::Anon }; - let feature_name = - Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), tcx.types.str_); - (0, 0, vec![feature_name], tcx.types.bool) - } + sym::target_feature_available_at_call_site => (0, 1, Vec::new(), tcx.types.bool), sym::const_eval_select => (4, 0, vec![param(0), param(1), param(2)], param(3)), diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 3bd0139fb565c..6e95a7bb88a86 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3691,9 +3691,9 @@ pub fn return_address() -> *const () { /// to support negligible or zero overhead branching on feature availability, for scenarios where /// runtime detection has too much overhead and globally enabled features are not sufficient. /// -/// Callers should pass a string literal naming a Rust target feature, such as `"avx"` or `"neon"`. -/// The compiler may accept a few additional simple constant forms, but this intrinsic should be -/// treated as taking a string literal. +/// This intrinsic takes a null-terminated feature name passed as a byte array. For a more +/// ergonomic interface, use the [`target_feature_available_at_call_site!`] macro, which accepts +/// a string literal. /// /// The LLVM backend implements this intrinsic by lowering to a marker that is resolved by a /// post-inlining pass. @@ -3711,4 +3711,34 @@ pub fn return_address() -> *const () { #[rustc_nounwind] #[unstable(feature = "core_intrinsics", issue = "none")] #[rustc_intrinsic] -pub fn target_feature_available_at_call_site(feature: &str) -> bool; +pub fn target_feature_available_at_call_site() -> bool; + +/// Returns whether a target feature is enabled at the call site. +/// +/// This macro is a more convenient interface for [`target_feature_available_at_call_site`]. +#[unstable(feature = "core_intrinsics", issue = "none")] +#[allow_internal_unstable(adt_const_params)] +#[diagnostic::on_unmatched_args( + note = "this macro expects a string literal target feature name, like `target_feature_available_at_call_site!(\"avx\")`" +)] +#[rustc_macro_transparency = "semiopaque"] +pub macro target_feature_available_at_call_site($feature:literal) {{ + ::core::intrinsics::target_feature_available_at_call_site::< + { + const { + let bytes = $feature.as_bytes(); + + assert!(bytes.len() <= 100, "feature string too long"); + + let mut out = [0u8; 100]; + let mut i = 0; + while i < bytes.len() { + out[i] = bytes[i]; + i += 1; + } + + out + } + }, + >() +}} diff --git a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs index e89a199bbe632..141fc85c4ae9e 100644 --- a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs +++ b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs @@ -1,6 +1,5 @@ //@ assembly-output: emit-asm -//@ compile-flags: --crate-type=lib -Copt-level=3 -C llvm-args=-x86-asm-syntax=intel --target=x86_64-unknown-linux-gnu -Ctarget-feature=-avx,-fma -//@ needs-llvm-components: x86 +//@ compile-flags: --crate-type=lib -Copt-level=3 -C llvm-args=-x86-asm-syntax=intel -Ctarget-feature=-avx,-fma //@ only-x86_64 //@ ignore-sgx //@ ignore-backends: gcc @@ -12,7 +11,7 @@ use std::intrinsics::target_feature_available_at_call_site; #[inline(always)] fn maybe_fma(x: f32, y: f32, z: f32) -> f32 { - if target_feature_available_at_call_site("fma") { x.mul_add(y, z) } else { x * y + z } + if target_feature_available_at_call_site!("fma") { x.mul_add(y, z) } else { x * y + z } } #[inline(always)] @@ -21,7 +20,7 @@ fn maybe_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { let x = _mm_set_ss(x); let y = _mm_set_ss(y); let z = _mm_set_ss(z); - let result = if target_feature_available_at_call_site("fma") { + let result = if target_feature_available_at_call_site!("fma") { _mm_fmadd_ss(x, y, z) } else { _mm_add_ss(_mm_mul_ss(x, y), z) diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs index cbb5936011365..4426d3caeae2a 100644 --- a/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs +++ b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs @@ -1,9 +1,9 @@ //@ revisions: NEG_ONLY NEG_POS POS_NEG -//@ compile-flags: --crate-type=lib -Copt-level=0 --target=x86_64-unknown-linux-gnu +//@ compile-flags: --crate-type=lib -Copt-level=0 //@ [NEG_ONLY] compile-flags: -Ctarget-feature=-fma //@ [NEG_POS] compile-flags: -Ctarget-feature=-fma,+fma //@ [POS_NEG] compile-flags: -Ctarget-feature=+fma,-fma -//@ needs-llvm-components: x86 +//@ only-x86_64 //@ ignore-backends: gcc // opt-level=0 has different inlining properties, so ensure that the pass still works. @@ -24,7 +24,7 @@ use std::intrinsics::target_feature_available_at_call_site; #[no_mangle] #[target_feature(enable = "fma")] pub fn with_fma() -> bool { - target_feature_available_at_call_site("fma") + target_feature_available_at_call_site!("fma") } // NEG_ONLY-LABEL: @without_fma( @@ -38,5 +38,5 @@ pub fn with_fma() -> bool { // POS_NEG: ret i1 false #[no_mangle] pub fn without_fma() -> bool { - target_feature_available_at_call_site("fma") + target_feature_available_at_call_site!("fma") } diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs index a55bf028d4eca..f42b3d9ca2203 100644 --- a/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs +++ b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs @@ -1,19 +1,14 @@ -//@ add-minicore -//@ compile-flags: --crate-type=lib -Copt-level=3 --target=aarch64-unknown-linux-gnu -//@ needs-llvm-components: aarch64 +//@ compile-flags: --crate-type=lib -Copt-level=3 +//@ only-aarch64 //@ ignore-backends: gcc // Test intrinsic on a feature that doesn't map to an LLVM feature. // Can't resolve after inlining, but can get the current function's // feature presence. -#![feature(no_core, lang_items, intrinsics)] -#![no_core] +#![feature(core_intrinsics)] -extern crate minicore; - -#[rustc_intrinsic] -pub fn target_feature_available_at_call_site(feature: &str) -> bool; +use std::intrinsics::target_feature_available_at_call_site; // CHECK-LABEL: @with_tme( // CHECK-NOT: rust.target_feature_available_at_call_site @@ -21,7 +16,7 @@ pub fn target_feature_available_at_call_site(feature: &str) -> bool; #[no_mangle] #[target_feature(enable = "tme")] pub fn with_tme() -> bool { - target_feature_available_at_call_site("tme") + target_feature_available_at_call_site!("tme") } // CHECK-LABEL: @without_tme( @@ -29,5 +24,5 @@ pub fn with_tme() -> bool { // CHECK: ret i1 false #[no_mangle] pub fn without_tme() -> bool { - target_feature_available_at_call_site("tme") + target_feature_available_at_call_site!("tme") } diff --git a/tests/codegen-llvm/target-feature-available-at-call-site.rs b/tests/codegen-llvm/target-feature-available-at-call-site.rs index e1afbd1f5b4bd..a6fabf613d33c 100644 --- a/tests/codegen-llvm/target-feature-available-at-call-site.rs +++ b/tests/codegen-llvm/target-feature-available-at-call-site.rs @@ -1,5 +1,5 @@ -//@ compile-flags: --crate-type=lib -Copt-level=3 --target=x86_64-unknown-linux-gnu -Ctarget-feature=-avx -//@ needs-llvm-components: x86 +//@ compile-flags: --crate-type=lib -Copt-level=3 -Ctarget-feature=-avx +//@ only-x86_64 //@ ignore-backends: gcc #![feature(core_intrinsics)] @@ -8,7 +8,7 @@ use std::intrinsics::target_feature_available_at_call_site; #[inline] pub fn avx_branch_value() -> i32 { - if target_feature_available_at_call_site("avx") { 1 } else { 0 } + if target_feature_available_at_call_site!("avx") { 1 } else { 0 } } // CHECK-LABEL: @with_avx( diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs index d9c177e68ea19..c26d003a01f07 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs @@ -1,6 +1,6 @@ #![feature(core_intrinsics)] -const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site("fma"); +const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site!("fma"); //~^ ERROR cannot call non-const intrinsic `target_feature_available_at_call_site` in constants fn main() { diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr index 9f843454251f6..2a0dfc481dc01 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr @@ -1,8 +1,10 @@ error: cannot call non-const intrinsic `target_feature_available_at_call_site` in constants --> $DIR/target-feature-available-at-call-site-const.rs:3:23 | -LL | const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site("fma"); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +LL | const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site!("fma"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `std::intrinsics::target_feature_available_at_call_site` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs index 3b4c19f3b1e56..8e999d278820a 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs @@ -1,5 +1,4 @@ -//@ compile-flags: --target=x86_64-unknown-linux-gnu --crate-type=lib --emit=llvm-ir -//@ needs-llvm-components: x86 +//@ compile-flags: --crate-type=lib --emit=llvm-ir //@ ignore-backends: gcc #![feature(core_intrinsics)] @@ -10,6 +9,6 @@ use std::intrinsics::target_feature_available_at_call_site; #[no_mangle] pub fn check() { let feature = black_box("fma"); - let _ = target_feature_available_at_call_site(feature); - //~^ ERROR `target_feature_available_at_call_site` requires a string literal argument + let _ = target_feature_available_at_call_site!(feature); + //~^ ERROR no rules expected `feature` } diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr index 5e036d21c3cee..175c534f15ee3 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr @@ -1,8 +1,12 @@ -error: `target_feature_available_at_call_site` requires a string literal argument - --> $DIR/target-feature-available-at-call-site-nonconst.rs:13:13 +error: no rules expected `feature` + --> $DIR/target-feature-available-at-call-site-nonconst.rs:12:52 | -LL | let _ = target_feature_available_at_call_site(feature); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +LL | let _ = target_feature_available_at_call_site!(feature); + | ^^^^^^^ no rules expected this token in macro call + | +note: while trying to match meta-variable `$feature:literal` + --> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL + = note: this macro expects a string literal target feature name, like `target_feature_available_at_call_site!("avx")` error: aborting due to 1 previous error diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs index f577ef33d63d3..048f413da1198 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs @@ -1,5 +1,4 @@ -//@ compile-flags: --target=x86_64-unknown-linux-gnu --crate-type=lib --emit=llvm-ir -//@ needs-llvm-components: x86 +//@ compile-flags: --crate-type=lib --emit=llvm-ir //@ ignore-backends: gcc #![feature(core_intrinsics)] @@ -9,6 +8,6 @@ use std::intrinsics::target_feature_available_at_call_site; #[no_mangle] pub fn check() { - let _ = target_feature_available_at_call_site("definitely-not-a-feature"); + let _ = target_feature_available_at_call_site!("definitely-not-a-feature"); //~^ ERROR unknown target feature `definitely-not-a-feature` } diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr index 46ddf1185d139..ec86dc4d01b69 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr @@ -1,8 +1,10 @@ error: unknown target feature `definitely-not-a-feature` - --> $DIR/target-feature-available-at-call-site-unknown.rs:12:13 + --> $DIR/target-feature-available-at-call-site-unknown.rs:11:13 | -LL | let _ = target_feature_available_at_call_site("definitely-not-a-feature"); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +LL | let _ = target_feature_available_at_call_site!("definitely-not-a-feature"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `target_feature_available_at_call_site` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error From 8170174d01bfae33272b6ffeea98c5b23a901e31 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 3 Jul 2026 23:03:30 -0400 Subject: [PATCH 5/9] Use metadata node rather than argument --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 10 ++++++---- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 -- compiler/rustc_codegen_llvm/src/type_.rs | 4 ---- compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 12 +++++++----- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4a6702055f9f5..330e7561b7854 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -951,7 +951,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } // Generate the marker to be replaced by the LLVM pass - let fn_ty = self.type_func(&[self.cx.type_metadata()], self.type_i1()); + let fn_ty = self.type_func(&[], self.type_i1()); let llfn = self.cx.declare_cfn( "rust.target_feature_available_at_call_site", llvm::UnnamedAddr::No, @@ -959,9 +959,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ); let nounwind = llvm::AttributeKind::NoUnwind.create_attr(self.cx.llcx); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[nounwind]); - let feature_arg = - self.cx.get_metadata_value(self.cx.create_metadata(llvm_feature_name.as_bytes())); - self.call(fn_ty, None, None, llfn, &[feature_arg], None, None) + let call = self.call(fn_ty, None, None, llfn, &[], None, None); + let kind = self.cx.get_md_kind_id("rust.target_feature"); + let feature = self.cx.create_metadata(llvm_feature_name.as_bytes()); + self.cx.set_metadata_node(call, kind, &[feature]); + call } fn codegen_llvm_intrinsic_call( diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6285eb23a6ff4..7a8cca7f37474 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -944,8 +944,6 @@ unsafe extern "C" { pub(crate) fn LLVMFloatTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; - pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type; - // Operations on non-IEEE real types pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 8001b917026e9..0d49971f52533 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -59,10 +59,6 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { unsafe { llvm::LLVMVoidTypeInContext(self.llcx()) } } - pub(crate) fn type_metadata(&self) -> &'ll Type { - unsafe { llvm::LLVMMetadataTypeInContext(self.llcx()) } - } - ///x Creates an integer type with the given number of bits, e.g., i24 pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { llvm::LLVMIntTypeInContext(self.llcx(), num_bits as c_uint) diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 8e8d14cdb658d..4e3a4dff96745 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -140,17 +140,19 @@ class TargetFeatureAvailableAtCallSitePass if (MarkerDecl == nullptr) return PreservedAnalyses::all(); + auto FeatureKindID = M.getContext().getMDKindID("rust.target_feature"); + SmallVector CallsToErase; for (User *U : MarkerDecl->users()) { auto *Call = dyn_cast(U); - if (!Call || Call->getCalledFunction() != MarkerDecl || - Call->arg_size() != 1) + if (!Call || Call->getCalledFunction() != MarkerDecl) continue; - auto *FeatureAsValue = dyn_cast(Call->getArgOperand(0)); + auto *FeatureNode = Call->getMetadata(FeatureKindID); auto *FeatureMetadata = - FeatureAsValue ? dyn_cast(FeatureAsValue->getMetadata()) - : nullptr; + FeatureNode && FeatureNode->getNumOperands() == 1 + ? dyn_cast(FeatureNode->getOperand(0)) + : nullptr; if (FeatureMetadata == nullptr) continue; From 2ebc988441d54bfa32b5a93cc1a5cdaf62326f04 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 3 Jul 2026 23:10:18 -0400 Subject: [PATCH 6/9] Remove unnecessary const block --- library/core/src/intrinsics/mod.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 6e95a7bb88a86..bbe80dce012ec 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3725,20 +3725,18 @@ pub fn target_feature_available_at_call_site() -> bool pub macro target_feature_available_at_call_site($feature:literal) {{ ::core::intrinsics::target_feature_available_at_call_site::< { - const { - let bytes = $feature.as_bytes(); + let bytes = $feature.as_bytes(); - assert!(bytes.len() <= 100, "feature string too long"); + assert!(bytes.len() <= 100, "feature string too long"); - let mut out = [0u8; 100]; - let mut i = 0; - while i < bytes.len() { - out[i] = bytes[i]; - i += 1; - } - - out + let mut out = [0u8; 100]; + let mut i = 0; + while i < bytes.len() { + out[i] = bytes[i]; + i += 1; } + + out }, >() }} From 50fb468297762b8e7893986900fcf06f0342ea16 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 5 Jul 2026 12:27:31 -0400 Subject: [PATCH 7/9] Simpler const hack --- library/core/src/intrinsics/mod.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index bbe80dce012ec..14fa3b32f9754 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3715,7 +3715,7 @@ pub fn target_feature_available_at_call_site() -> bool /// Returns whether a target feature is enabled at the call site. /// -/// This macro is a more convenient interface for [`target_feature_available_at_call_site`]. +/// This macro is a more convenient interface for [`target_feature_available_at_call_site()`]. #[unstable(feature = "core_intrinsics", issue = "none")] #[allow_internal_unstable(adt_const_params)] #[diagnostic::on_unmatched_args( @@ -3729,12 +3729,10 @@ pub macro target_feature_available_at_call_site($feature:literal) {{ assert!(bytes.len() <= 100, "feature string too long"); + // FIXME(const-hack) can't use subslicing yet let mut out = [0u8; 100]; - let mut i = 0; - while i < bytes.len() { - out[i] = bytes[i]; - i += 1; - } + let (dst, _) = out.split_at_mut(bytes.len()); + dst.copy_from_slice(bytes); out }, From c63180b0a98c04c16924c009e795be0dbfe7bdb2 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 5 Jul 2026 14:24:02 -0400 Subject: [PATCH 8/9] Move intrinsic to core::intrinsics::simd --- library/core/src/intrinsics/mod.rs | 59 ------------------- library/core/src/intrinsics/simd/mod.rs | 59 +++++++++++++++++++ ...rget-feature-available-at-call-site-fma.rs | 2 +- ...arget-feature-available-at-call-site-o0.rs | 2 +- ...vailable-at-call-site-rust-only-feature.rs | 2 +- .../target-feature-available-at-call-site.rs | 2 +- ...et-feature-available-at-call-site-const.rs | 2 +- ...eature-available-at-call-site-const.stderr | 6 +- ...feature-available-at-call-site-nonconst.rs | 2 +- ...ure-available-at-call-site-nonconst.stderr | 2 +- ...-feature-available-at-call-site-unknown.rs | 2 +- 11 files changed, 70 insertions(+), 70 deletions(-) diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 14fa3b32f9754..bc96c768c0c94 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3679,62 +3679,3 @@ pub const unsafe fn va_end(ap: &mut VaList<'_>) { pub fn return_address() -> *const () { core::ptr::null() } - -/// Returns whether the named target feature is available at the current call site. -/// This will not emit any runtime detection code; it always turns into `true` or `false` at some -/// point during compilation. -/// -/// "Available" in this context means known to be enabled in the calling function. -/// At a minimum, being enabled globally or in `#[target_features]` of the function containing -/// this intrinsic is considered available. Depending on codegen options and backend, the features -/// of the function the intrinsic call is inlined into may also be accounted for. This is intended -/// to support negligible or zero overhead branching on feature availability, for scenarios where -/// runtime detection has too much overhead and globally enabled features are not sufficient. -/// -/// This intrinsic takes a null-terminated feature name passed as a byte array. For a more -/// ergonomic interface, use the [`target_feature_available_at_call_site!`] macro, which accepts -/// a string literal. -/// -/// The LLVM backend implements this intrinsic by lowering to a marker that is resolved by a -/// post-inlining pass. -/// -/// # Safety -/// -/// The return value of this intrinsic is call-site dependent and may vary between calls to the -/// same function, depending on inlining and codegen. Callers must not rely on the result being -/// stable across calls or multiple uses of the same function containing the intrinsic. -/// -/// A return value of `true` guarantees that the feature is enabled at the call site and it is -/// safe to call other functions that require the feature. -/// A return value of `false` carries no information and does not indicate whether the feature is -/// enabled or not. -#[rustc_nounwind] -#[unstable(feature = "core_intrinsics", issue = "none")] -#[rustc_intrinsic] -pub fn target_feature_available_at_call_site() -> bool; - -/// Returns whether a target feature is enabled at the call site. -/// -/// This macro is a more convenient interface for [`target_feature_available_at_call_site()`]. -#[unstable(feature = "core_intrinsics", issue = "none")] -#[allow_internal_unstable(adt_const_params)] -#[diagnostic::on_unmatched_args( - note = "this macro expects a string literal target feature name, like `target_feature_available_at_call_site!(\"avx\")`" -)] -#[rustc_macro_transparency = "semiopaque"] -pub macro target_feature_available_at_call_site($feature:literal) {{ - ::core::intrinsics::target_feature_available_at_call_site::< - { - let bytes = $feature.as_bytes(); - - assert!(bytes.len() <= 100, "feature string too long"); - - // FIXME(const-hack) can't use subslicing yet - let mut out = [0u8; 100]; - let (dst, _) = out.split_at_mut(bytes.len()); - dst.copy_from_slice(bytes); - - out - }, - >() -}} diff --git a/library/core/src/intrinsics/simd/mod.rs b/library/core/src/intrinsics/simd/mod.rs index 9311dcc9bd00e..ecfa8c3239071 100644 --- a/library/core/src/intrinsics/simd/mod.rs +++ b/library/core/src/intrinsics/simd/mod.rs @@ -839,3 +839,62 @@ pub unsafe fn simd_flog2(a: T) -> T; #[rustc_intrinsic] #[rustc_nounwind] pub unsafe fn simd_flog(a: T) -> T; + +/// Returns whether the named target feature is available at the current call site. +/// This will not emit any runtime detection code; it always turns into `true` or `false` at some +/// point during compilation. +/// +/// "Available" in this context means known to be enabled in the calling function. +/// At a minimum, being enabled globally or in `#[target_features]` of the function containing +/// this intrinsic is considered available. Depending on codegen options and backend, the features +/// of the function the intrinsic call is inlined into may also be accounted for. This is intended +/// to support negligible or zero overhead branching on feature availability, for scenarios where +/// runtime detection has too much overhead and globally enabled features are not sufficient. +/// +/// This intrinsic takes a null-terminated feature name passed as a byte array. For a more +/// ergonomic interface, use the [`target_feature_available_at_call_site!`] macro, which accepts +/// a string literal. +/// +/// The LLVM backend implements this intrinsic by lowering to a marker that is resolved by a +/// post-inlining pass. +/// +/// # Safety +/// +/// The return value of this intrinsic is call-site dependent and may vary between calls to the +/// same function, depending on inlining and codegen. Callers must not rely on the result being +/// stable across calls or multiple uses of the same function containing the intrinsic. +/// +/// A return value of `true` guarantees that the feature is enabled at the call site and it is +/// safe to call other functions that require the feature. +/// A return value of `false` carries no information and does not indicate whether the feature is +/// enabled or not. +#[rustc_nounwind] +#[unstable(feature = "core_intrinsics", issue = "none")] +#[rustc_intrinsic] +pub fn target_feature_available_at_call_site() -> bool; + +/// Returns whether a target feature is enabled at the call site. +/// +/// This macro is a more convenient interface for [`target_feature_available_at_call_site()`]. +#[unstable(feature = "core_intrinsics", issue = "none")] +#[allow_internal_unstable(adt_const_params)] +#[diagnostic::on_unmatched_args( + note = "this macro expects a string literal target feature name, like `target_feature_available_at_call_site!(\"avx\")`" +)] +#[rustc_macro_transparency = "semiopaque"] +pub macro target_feature_available_at_call_site($feature:literal) {{ + ::core::intrinsics::simd::target_feature_available_at_call_site::< + { + let bytes = $feature.as_bytes(); + + assert!(bytes.len() <= 100, "feature string too long"); + + // FIXME(const-hack) can't use subslicing yet + let mut out = [0u8; 100]; + let (dst, _) = out.split_at_mut(bytes.len()); + dst.copy_from_slice(bytes); + + out + }, + >() +}} diff --git a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs index 141fc85c4ae9e..ec4501db22cb7 100644 --- a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs +++ b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs @@ -7,7 +7,7 @@ #![feature(core_intrinsics)] use std::arch::x86_64::{_mm_add_ss, _mm_cvtss_f32, _mm_fmadd_ss, _mm_mul_ss, _mm_set_ss}; -use std::intrinsics::target_feature_available_at_call_site; +use std::intrinsics::simd::target_feature_available_at_call_site; #[inline(always)] fn maybe_fma(x: f32, y: f32, z: f32) -> f32 { diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs index 4426d3caeae2a..ec08047bbe62e 100644 --- a/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs +++ b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs @@ -10,7 +10,7 @@ #![feature(core_intrinsics)] -use std::intrinsics::target_feature_available_at_call_site; +use std::intrinsics::simd::target_feature_available_at_call_site; // NEG_ONLY-LABEL: @with_fma( // NEG_ONLY-NOT: rust.target_feature_available_at_call_site diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs index f42b3d9ca2203..e804224b86353 100644 --- a/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs +++ b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs @@ -8,7 +8,7 @@ #![feature(core_intrinsics)] -use std::intrinsics::target_feature_available_at_call_site; +use std::intrinsics::simd::target_feature_available_at_call_site; // CHECK-LABEL: @with_tme( // CHECK-NOT: rust.target_feature_available_at_call_site diff --git a/tests/codegen-llvm/target-feature-available-at-call-site.rs b/tests/codegen-llvm/target-feature-available-at-call-site.rs index a6fabf613d33c..6a685e430132e 100644 --- a/tests/codegen-llvm/target-feature-available-at-call-site.rs +++ b/tests/codegen-llvm/target-feature-available-at-call-site.rs @@ -4,7 +4,7 @@ #![feature(core_intrinsics)] -use std::intrinsics::target_feature_available_at_call_site; +use std::intrinsics::simd::target_feature_available_at_call_site; #[inline] pub fn avx_branch_value() -> i32 { diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs index c26d003a01f07..3d7625709d59e 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs @@ -1,6 +1,6 @@ #![feature(core_intrinsics)] -const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site!("fma"); +const HAS_FMA: bool = std::intrinsics::simd::target_feature_available_at_call_site!("fma"); //~^ ERROR cannot call non-const intrinsic `target_feature_available_at_call_site` in constants fn main() { diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr index 2a0dfc481dc01..a88d842c879da 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr @@ -1,10 +1,10 @@ error: cannot call non-const intrinsic `target_feature_available_at_call_site` in constants --> $DIR/target-feature-available-at-call-site-const.rs:3:23 | -LL | const HAS_FMA: bool = std::intrinsics::target_feature_available_at_call_site!("fma"); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +LL | const HAS_FMA: bool = std::intrinsics::simd::target_feature_available_at_call_site!("fma"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | - = note: this error originates in the macro `std::intrinsics::target_feature_available_at_call_site` (in Nightly builds, run with -Z macro-backtrace for more info) + = note: this error originates in the macro `std::intrinsics::simd::target_feature_available_at_call_site` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs index 8e999d278820a..7e10c69ccf4c1 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs @@ -4,7 +4,7 @@ #![feature(core_intrinsics)] use std::hint::black_box; -use std::intrinsics::target_feature_available_at_call_site; +use std::intrinsics::simd::target_feature_available_at_call_site; #[no_mangle] pub fn check() { diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr index 175c534f15ee3..61328c6ed8a5d 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr @@ -5,7 +5,7 @@ LL | let _ = target_feature_available_at_call_site!(feature); | ^^^^^^^ no rules expected this token in macro call | note: while trying to match meta-variable `$feature:literal` - --> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL + --> $SRC_DIR/core/src/intrinsics/simd/mod.rs:LL:COL = note: this macro expects a string literal target feature name, like `target_feature_available_at_call_site!("avx")` error: aborting due to 1 previous error diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs index 048f413da1198..595285c971b7a 100644 --- a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs @@ -4,7 +4,7 @@ #![feature(core_intrinsics)] use std::hint::black_box; -use std::intrinsics::target_feature_available_at_call_site; +use std::intrinsics::simd::target_feature_available_at_call_site; #[no_mangle] pub fn check() { From 81bc0addd8e8c13d52326400b489b7cc17971f3b Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 5 Jul 2026 22:16:05 -0400 Subject: [PATCH 9/9] Refactor intrinsic to implement for cg_clif --- .../src/intrinsics/mod.rs | 25 +++++++ .../rustc_codegen_ssa/src/mir/intrinsic.rs | 75 ++++--------------- .../rustc_codegen_ssa/src/target_features.rs | 64 +++++++++++++++- 3 files changed, 104 insertions(+), 60 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index e6b8c537d8451..8a4ebfc360ddc 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -20,6 +20,8 @@ mod simd; use cranelift_codegen::ir::{ AtomicRmwOp, BlockArg, ExceptionTableData, ExceptionTableItem, ExceptionTag, }; +use rustc_codegen_ssa::target_features; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::ty; use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::layout::ValidityRequirement; @@ -834,6 +836,29 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, caller_location); } + sym::target_feature_available_at_call_site => { + intrinsic_args!(fx, args => (); intrinsic); + + let enabled = match target_features::target_feature_available_at_call_site_intrinsic( + fx.tcx, + source_info.span, + fx.instance.def_id(), + generic_args, + ) { + target_features::TargetFeatureAvailableAtCallSite::Known(enabled) => enabled, + // SSA already handles the easy case where the caller function has the feature enabled. + // Cranelift doesn't (yet) have the equivalent LLVM pass to replace a marker post-inlining, + // so report that the feature is unavailable at the call site. + target_features::TargetFeatureAvailableAtCallSite::CheckBackend(_) => false, + }; + + let ret_val = CValue::by_val( + fx.bcx.ins().iconst(types::I8, i64::from(enabled)), + fx.layout_of(fx.tcx.types.bool), + ); + ret.write_cvalue(fx, ret_val); + } + _ if intrinsic.as_str().starts_with("atomic_fence") => { intrinsic_args!(fx, args => (); intrinsic); diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 19ffb437e1850..64b2cb5229774 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,10 +1,9 @@ use rustc_abi::{Align, FieldIdx, WrappingRange}; -use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::mir::{self, SourceInfo}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; -use rustc_span::{ErrorGuaranteed, Symbol, sym}; +use rustc_span::{ErrorGuaranteed, sym}; use rustc_target::spec::Arch; use super::operand::{OperandRef, OperandValue}; @@ -14,7 +13,7 @@ use crate::common::{AtomicRmwBinOp, SynchronizationScope}; use crate::errors::InvalidMonomorphization; use crate::mir::operand::OperandRefBuilder; use crate::traits::*; -use crate::{MemFlags, meth, size_of_val}; +use crate::{MemFlags, meth, size_of_val, target_features}; fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, @@ -593,62 +592,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } sym::target_feature_available_at_call_site => { - // This intrinsic either returns a bool reflecting feature presence, or a marker. A later - // LLVM pass will replace the marker with the feature presence. - // * If the feature is already known to be present, we can immediately return true. - // * If we can detect the feature later in the LLVM pass, return the marker. - // * If the feature isn't present and we can't detect it later, return false. - - // Return false on errors to allow diagnostics to continue - fn err_false<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - span: rustc_span::Span, - msg: impl Into, - ) -> IntrinsicResult<'tcx, Bx::Value> { - bx.tcx().dcx().span_err(span, msg); - IntrinsicResult::Operand(OperandValue::Immediate(bx.const_bool(false))) - } - - let Some(feature_bytes) = fn_args - .const_at(0) - .try_to_value() - .and_then(|feature| feature.try_to_raw_bytes(bx.tcx())) - else { - span_bug!(span, "wrong const argument for {name}"); - }; - let feature_len = - feature_bytes.iter().position(|&byte| byte == 0).unwrap_or(feature_bytes.len()); - let Some(feature_name) = std::str::from_utf8(&feature_bytes[..feature_len]).ok() - else { - return err_false( - bx, - span, - "`target_feature_available_at_call_site` requires a UTF-8 feature name", - ); - }; - - // Ensure it's a valid Rust feature - let rust_target_features = bx.tcx().rust_target_features(LOCAL_CRATE); - let Some(&stability) = rust_target_features.get(feature_name) else { - return err_false(bx, span, format!("unknown target feature `{feature_name}`")); - }; - if let Err(reason) = stability.toggle_allowed() { - return err_false( - bx, - span, - format!("cannot use target feature `{feature_name}`: {reason}"), - ); - } - - // If the function has the feature, we can emit true now. Otherwise emit the marker. - let current_fn_features = - crate::target_features::asm_target_features(bx.tcx(), self.instance.def_id()); - if current_fn_features.contains(&Symbol::intern(feature_name)) { - OperandValue::Immediate(bx.const_bool(true)) - } else { - OperandValue::Immediate( - bx.codegen_target_feature_available_at_call_site(feature_name), - ) + match target_features::target_feature_available_at_call_site_intrinsic( + bx.tcx(), + span, + self.instance.def_id(), + fn_args, + ) { + target_features::TargetFeatureAvailableAtCallSite::Known(enabled) => { + OperandValue::Immediate(bx.const_bool(enabled)) + } + target_features::TargetFeatureAvailableAtCallSite::CheckBackend(feature) => { + OperandValue::Immediate( + bx.codegen_target_feature_available_at_call_site(feature.as_str()), + ) + } } } diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index d02c6e43a85a5..128303d28334a 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -5,7 +5,8 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId}; use rustc_middle::middle::codegen_fn_attrs::{TargetFeature, TargetFeatureKind}; use rustc_middle::query::Providers; -use rustc_middle::ty::TyCtxt; +use rustc_middle::span_bug; +use rustc_middle::ty::{self, TyCtxt}; use rustc_session::Session; use rustc_session::errors::feature_err; use rustc_session::lint::builtin::AARCH64_SOFTFLOAT_NEON; @@ -569,3 +570,64 @@ pub(crate) fn provide(providers: &mut Providers) { ..*providers } } + +#[derive(Copy, Clone, Debug)] +pub enum TargetFeatureAvailableAtCallSite { + /// The feature's presence is known immediately, including `false` for error conditions. + Known(bool), + /// It's not known if the feature is available at the call site. It's safe to return `false`, + /// but the backend may be able to refine the answer (e.g. the LLVM post-inline pass) + CheckBackend(Symbol), +} + +/// Implementation for the `target_feature_available_at_call_site` intrinsic. +/// +/// This function performs the backend-independent implementation: +/// * If the feature is enabled for the current function, we can immediately return true. +/// * If the feature is not enabled for the current function, defer to the backend to account +/// for inlining. +/// * If an error occurs, emit the diagnostic and return false. +/// +/// When deferred to the backend, it's always safe to return false, but optimization opportunities +/// are lost. +/// The LLVM backend, for example, emits a marker function that is replaced with true or false +/// with a post-inlining pass. +pub fn target_feature_available_at_call_site_intrinsic<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + did: DefId, + generic_args: ty::GenericArgsRef<'tcx>, +) -> TargetFeatureAvailableAtCallSite { + let Some(feature_bytes) = + generic_args.const_at(0).try_to_value().and_then(|feature| feature.try_to_raw_bytes(tcx)) + else { + span_bug!(span, "wrong const argument for target_feature_available_at_call_site"); + }; + + let feature_len = + feature_bytes.iter().position(|&byte| byte == 0).unwrap_or(feature_bytes.len()); + let Some(feature_name) = std::str::from_utf8(&feature_bytes[..feature_len]).ok() else { + tcx.dcx().span_err( + span, + "`target_feature_available_at_call_site` requires a UTF-8 feature name", + ); + return TargetFeatureAvailableAtCallSite::Known(false); + }; + let feature = Symbol::intern(feature_name); + + let rust_target_features = tcx.rust_target_features(LOCAL_CRATE); + let Some(&stability) = rust_target_features.get(feature_name) else { + tcx.dcx().span_err(span, format!("unknown target feature `{feature}`")); + return TargetFeatureAvailableAtCallSite::Known(false); + }; + if let Err(reason) = stability.toggle_allowed() { + tcx.dcx().span_err(span, format!("cannot use target feature `{feature}`: {reason}")); + return TargetFeatureAvailableAtCallSite::Known(false); + } + + if asm_target_features(tcx, did).contains(&feature) { + TargetFeatureAvailableAtCallSite::Known(true) + } else { + TargetFeatureAvailableAtCallSite::CheckBackend(feature) + } +}