[SYCL][AMDGPU] Set amdgpu-flat-work-group-size for SYCL reqd_work_group_size

.github/workflows/pr-code-format.yml

@@ -56,7 +56,7 @@ jobs:
       - name: Install clang-format
         uses: aminya/setup-cpp@1f17f92d6a52bfcb1a25348e2c526c2e5cbb1134 # v1.8.0
         with:
-          clangformat: 20.1.8
+          clangformat: true
 
       - name: Setup Python env
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0

.gitignore

@@ -11,6 +11,8 @@
 #==============================================================================#
 # Temp files created by most text editors.
 *~
+# Emacs lock files
+.#*
 # Merge files created by git.
 *.orig
 # Reject files created by patch.

bolt/docs/conf.py

@@ -13,31 +13,15 @@
 import sys, os
 from datetime import date
 
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-# sys.path.insert(0, os.path.abspath('.'))
+from llvm_sphinx import *  # see llvm-project/utils/docs/README.md
 
-# -- General configuration -----------------------------------------------------
+globals().update(common_conf(tags, markdown=Markdown.NEVER))
 
-# If your documentation needs a minimal Sphinx version, state it here.
-# needs_sphinx = '1.0'
+# -- General configuration -----------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ["sphinx.ext.intersphinx", "sphinx.ext.todo"]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
-# The suffix of source filenames.
-source_suffix = ".rst"
-
-# The encoding of source files.
-# source_encoding = 'utf-8-sig'
-
-# The master toctree document.
-master_doc = "index"
+extensions += ["sphinx.ext.intersphinx", "sphinx.ext.todo"]
 
 # General information about the project.
 project = "BOLT"

bolt/include/bolt/Core/BinaryContext.h

@@ -816,6 +816,22 @@ class BinaryContext {
   std::atomic<uint16_t> MaxMainCodeAlignment{1};
   std::atomic<uint16_t> MaxColdCodeAlignment{1};
 
+  /// Alignment-related options sourced from CommandLineOpts. Populated by
+  /// RewriteInstance::adjustCommandLineOptions() so passes and the emitter
+  /// can read them via BinaryContext instead of touching opts::* directly.
+  /// Defaults must stay in sync with the cl::init values in
+  /// bolt/lib/Utils/CommandLineOpts.cpp.
+  unsigned AlignText{0};
+  unsigned AlignFunctions{64};
+  unsigned AlignBlocksMinSize{0};
+  unsigned AlignBlocksThreshold{800};
+  unsigned AlignFunctionsMaxBytes{32};
+  unsigned BlockAlignment{16};
+  bool AlignBlocks{false};
+  bool PreserveBlocksAlignment{false};
+  bool UseCompactAligner{true};
+  bool X86AlignBranchBoundaryHotOnly{true};
+
   /// Fold \p Alignment into the running max for the main code section (when
   /// \p InMainSection) and/or the cold code section (when \p InColdSection),
   /// reflecting which output section(s) the object is emitted into. Safe to

bolt/include/bolt/Core/BinaryFunction.h

@@ -37,6 +37,7 @@
 #include "bolt/Utils/NameResolver.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
@@ -193,7 +194,9 @@ class BinaryFunction {
     /// Keeps track of other functions we depend on because there is a reference
     /// to the constant islands in them.
     IslandProxiesType Proxies, ColdProxies;
-    SmallPtrSet<BinaryFunction *, 1> Dependency; // The other way around
+    SetVector<BinaryFunction *, SmallVector<BinaryFunction *, 1>,
+              SmallPtrSet<BinaryFunction *, 1>>
+        Dependency; // The other way around
 
     mutable MCSymbol *FunctionConstantIslandLabel{nullptr};
     mutable MCSymbol *FunctionColdConstantIslandLabel{nullptr};
@@ -1022,6 +1025,14 @@ class BinaryFunction {
 
   std::optional<MCInst> disassembleInstructionAtOffset(uint64_t Offset) const;
 
+  /// Given a starting point \p Offset and a number of bytes \p MinLength,
+  /// returns the number of bytes \p MinLength + Tail such that the last
+  /// instruction in the sequence is not split apart. Returns std::nullopt if
+  /// disassembling fails. Assumes that \p Offset aligns with instruction stream
+  /// and that the instructions can be disassembled.
+  uint64_t getInstructionSequenceLength(uint64_t Offset,
+                                        uint64_t MinLength) const;
+
   /// Return offset for the first instruction. If there is data at the
   /// beginning of a function then offset of the first instruction could
   /// be different from 0

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1995,6 +1995,11 @@ class MCPlusBuilder {
     llvm_unreachable("not implemented");
   }
 
+  /// Creates a breakpoint instruction in Inst.
+  virtual void createBreakpoint(MCInst &Inst) const {
+    llvm_unreachable("not implemented");
+  }
+
   /// Creates an instruction to bump the stack pointer just like a call.
   virtual void createStackPointerIncrement(MCInst &Inst, int Size = 8,
                                            bool NoFlagsClobber = false) const {

bolt/include/bolt/Passes/PatchEntries.h

@@ -26,6 +26,7 @@ class PatchEntries : public BinaryFunctionPass {
   struct Patch {
     const MCSymbol *Symbol;
     uint64_t Address;
+    uint32_t PaddingAfter = 0;
   };
 
 public:

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -72,6 +72,14 @@ extern llvm::cl::OptionCategory BinaryAnalysisCategory;
 
 extern llvm::cl::opt<unsigned> AlignText;
 extern llvm::cl::opt<unsigned> AlignFunctions;
+extern llvm::cl::opt<bool> AlignBlocks;
+extern llvm::cl::opt<unsigned> AlignBlocksMinSize;
+extern llvm::cl::opt<unsigned> AlignBlocksThreshold;
+extern llvm::cl::opt<unsigned> AlignFunctionsMaxBytes;
+extern llvm::cl::opt<unsigned> BlockAlignment;
+extern llvm::cl::opt<bool> PreserveBlocksAlignment;
+extern llvm::cl::opt<bool> UseCompactAligner;
+extern llvm::cl::opt<bool> X86AlignBranchBoundaryHotOnly;
 extern llvm::cl::opt<bool> AggregateOnly;
 extern llvm::cl::opt<bool> ArmSPE;
 extern llvm::cl::opt<unsigned> BucketsPerLine;

bolt/lib/Core/BinaryEmitter.cpp

@@ -33,10 +33,6 @@ using namespace bolt;
 namespace opts {
 
 extern cl::opt<JumpTableSupportLevel> JumpTables;
-extern cl::opt<bool> PreserveBlocksAlignment;
-
-cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),
-                          cl::cat(BoltOptCategory));
 
 static cl::list<std::string>
 BreakFunctionNames("break-funcs",
@@ -68,12 +64,6 @@ static cl::opt<bool> PrintJumpTables("print-jump-tables",
                                      cl::desc("print jump tables"), cl::Hidden,
                                      cl::cat(BoltCategory));
 
-static cl::opt<bool>
-X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
-  cl::desc("only apply branch boundary alignment in hot code"),
-  cl::init(true),
-  cl::cat(BoltOptCategory));
-
 size_t padFunction(std::map<std::string, size_t> &FunctionPadding,
                    const cl::list<std::string> &Spec,
                    const BinaryFunction &Function) {
@@ -214,7 +204,7 @@ void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
   if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())
     RtLibrary->emitBinary(BC, Streamer);
 
-  BC.getTextSection()->setAlignment(Align(opts::AlignText));
+  BC.getTextSection()->setAlignment(Align(BC.AlignText));
 
   emitFunctions();
 
@@ -246,15 +236,15 @@ void BinaryEmitter::emitFunctions() {
       bool Emitted = false;
 
       // Turn off Intel JCC Erratum mitigation for cold code if requested
-      if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
+      if (HasProfile && BC.X86AlignBranchBoundaryHotOnly &&
           !Function->hasValidProfile())
         Streamer.setAllowAutoPadding(false);
 
       FunctionLayout &Layout = Function->getLayout();
       Emitted |= emitFunction(*Function, Layout.getMainFragment());
 
       if (Function->isSplit()) {
-        if (opts::X86AlignBranchBoundaryHotOnly)
+        if (BC.X86AlignBranchBoundaryHotOnly)
           Streamer.setAllowAutoPadding(false);
 
         assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
@@ -317,16 +307,16 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
     // Set section alignment to at least maximum possible object alignment.
     // We need this to support LongJmp and other passes that calculates
     // tentative layout.
-    Section->ensureMinAlignment(Align(opts::AlignFunctions));
+    Section->ensureMinAlignment(Align(BC.AlignFunctions));
 
-    Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);
+    Streamer.emitCodeAlignment(Function.getMinAlign(), *BC.STI);
     uint16_t MaxAlignBytes = FF.isSplitFragment()
                                  ? Function.getMaxColdAlignmentBytes()
                                  : Function.getMaxAlignmentBytes();
     if (MaxAlignBytes > 0)
-      Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes);
+      Streamer.emitCodeAlignment(Function.getAlign(), *BC.STI, MaxAlignBytes);
   } else {
-    Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI);
+    Streamer.emitCodeAlignment(Function.getAlign(), *BC.STI);
   }
 
   if (size_t Padding = opts::padFunctionBefore(Function)) {
@@ -457,9 +447,9 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
   // Track the first emitted instruction with debug info.
   bool FirstInstr = true;
   for (BinaryBasicBlock *const BB : FF) {
-    if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&
+    if ((BC.AlignBlocks || BC.PreserveBlocksAlignment) &&
         BB->getAlignment() > 1)
-      Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI,
+      Streamer.emitCodeAlignment(BB->getAlign(), *BC.STI,
                                  BB->getAlignmentMaxBytes());
     Streamer.emitLabel(BB->getLabel());
     if (!EmitCodeOnly) {
@@ -537,7 +527,7 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
   const uint16_t Alignment = OnBehalfOf
                                  ? OnBehalfOf->getConstantIslandAlignment()
                                  : BF.getConstantIslandAlignment();
-  Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI);
+  Streamer.emitCodeAlignment(Align(Alignment), *BC.STI);
 
   if (!OnBehalfOf) {
     if (!EmitColdPart)

bolt/lib/Core/BinaryFunction.cpp

@@ -111,11 +111,6 @@ static cl::opt<bool> NoScan(
         "slower binary)"),
     cl::Hidden, cl::cat(BoltOptCategory));
 
-cl::opt<bool>
-    PreserveBlocksAlignment("preserve-blocks-alignment",
-                            cl::desc("try to preserve basic block alignment"),
-                            cl::cat(BoltOptCategory));
-
 static cl::opt<bool> PrintOutputAddressRange(
     "print-output-address-range",
     cl::desc(
@@ -1284,6 +1279,29 @@ BinaryFunction::disassembleInstructionAtOffset(uint64_t Offset) const {
   return std::nullopt;
 }
 
+uint64_t
+BinaryFunction::getInstructionSequenceLength(uint64_t Offset,
+                                             uint64_t MinLength) const {
+  assert(Offset + MinLength <= MaxSize && "Invalid offset / min length");
+  ErrorOr<ArrayRef<unsigned char>> FunctionData = getData();
+  assert(FunctionData && "Cannot get function as data");
+  uint64_t Current = Offset;
+  const uint64_t Target = Offset + MinLength;
+  while (Current < Target) {
+    MCInst Instr;
+    uint64_t InstrSize = 0;
+    const uint64_t InstrAddress = getAddress() + Current;
+    [[maybe_unused]] MCDisassembler::DecodeStatus Res =
+        BC.DisAsm->getInstruction(Instr, InstrSize,
+                                  FunctionData->slice(Current), InstrAddress,
+                                  nulls());
+    assert(Res != MCDisassembler::DecodeStatus::Fail &&
+           "Function has been disassembled previously");
+    Current += InstrSize;
+  }
+  return Current - Offset;
+}
+
 Error BinaryFunction::disassemble() {
   NamedRegionTimer T("disassemble", "Disassemble function", "buildfuncs",
                      "Build Binary Functions", opts::TimeBuild);
@@ -2351,7 +2369,7 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
       // Always create new BB at branch destination.
       PrevBB = InsertBB ? InsertBB : PrevBB;
       InsertBB = addBasicBlockAt(LI->first, LI->second);
-      if (opts::PreserveBlocksAlignment && IsLastInstrNop)
+      if (BC.PreserveBlocksAlignment && IsLastInstrNop)
         InsertBB->setDerivedAlignment();
 
       if (PrevBB)
@@ -2388,7 +2406,7 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
           Label = BC.Ctx->createNamedTempSymbol("FT");
         }
         InsertBB = addBasicBlockAt(Offset, Label);
-        if (opts::PreserveBlocksAlignment && IsLastInstrNop)
+        if (BC.PreserveBlocksAlignment && IsLastInstrNop)
           InsertBB->setDerivedAlignment();
         updateOffset(LastInstrOffset);
       }