LLVM and SPIRV-LLVM-Translator pulldown (WW36 2026)

.github/new-prs-labeler.yml

@@ -935,7 +935,6 @@ backend:SPIR-V:
       - llvm/test/CodeGen/SPIRV/**
       - llvm/test/Frontend/HLSL/**
       - llvm/docs/SPIRVUsage.rst
-      - llvm/tools/spirv-tools/**
 
 mlgo:
   - changed-files:

bolt/docs/CommandLineArgumentReference.md

@@ -196,6 +196,11 @@
 
   Write BOLT Address Translation tables
 
+- `--fix-branches-with-liveness`
+
+  Use liveness analysis to decide whether AArch64 branch inversion must
+  preserve condition flags
+
 - `--force-data-relocations`
 
   Force relocations to data sections to always be processed

bolt/include/bolt/Core/BinaryContext.h

@@ -23,8 +23,10 @@
 #include "bolt/RuntimeLibs/RuntimeLibrary.h"
 #include "llvm/ADT/AddressRanges.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -261,15 +263,18 @@ class BinaryContext {
   /// The runtime library.
   std::unique_ptr<RuntimeLibrary> RtLibrary;
 
-  /// DWP Context.
-  std::shared_ptr<DWARFContext> DWPContext;
-
   /// Decoded pseudo probes.
   std::shared_ptr<MCPseudoProbeDecoder> PseudoProbeDecoder;
 
-  /// A map of DWO Ids to CUs.
+  /// Populated once in preprocessDWODebugInfo() and immutable thereafter; it
+  /// lets getDWOCU() (re-)open a DWO context on demand. The context is then
+  /// owned and cached by the skeleton unit itself (DWARFUnit::hasDWO()).
   using DWOIdToCUMapType = std::unordered_map<uint64_t, DWARFUnit *>;
-  DWOIdToCUMapType DWOCUs;
+  DWOIdToCUMapType DWOIdToSkeletonCU;
+
+  /// With a package every split CU shares one DWARFContext, which changes how
+  /// it may be released -- see releaseDWOCU().
+  bool UsesDWP{false};
 
   bool ContainsDwarf5{false};
   bool ContainsDwarfLegacy{false};
@@ -326,14 +331,32 @@ class BinaryContext {
 
   void clearFragmentsToSkip() { FragmentsToSkip.clear(); }
 
-  /// Given DWOId returns CU if it exists in DWOCUs.
+  /// True if split-dwarf files being processed come from a package (as
+  /// opposed to dwo files scattered on disk).
+  bool usesDWP() const { return UsesDWP; }
+
+  /// Given a DWOId, return the corresponding split CU, lazily opening its DWO
+  /// context if needed. Returns std::nullopt if the DWO could not be loaded.
+  ///
+  /// There is no shared mutable state to guard here: DWOIdToSkeletonCU is
+  /// immutable after preprocessing and the context is cached by the skeleton
+  /// unit. Callers must, however, keep a given DWOId to a single thread at a
+  /// time -- during parallel rewriting each DWOId belongs to exactly one
+  /// bucket -- because opening and releasing both mutate the skeleton unit's
+  /// DWO pointer.
   std::optional<DWARFUnit *> getDWOCU(uint64_t DWOId);
 
-  /// Returns DWOContext if it exists.
-  DWARFContext *getDWOContext() const;
+  /// Release the DWO context previously opened for \p DWOId (if any), freeing
+  /// its DWARFContext and parsed unit vector. Same threading contract as
+  /// getDWOCU().
+  void releaseDWOCU(uint64_t DWOId);
+
+  /// Release all currently-open DWO contexts. Used after preprocessing, so that
+  /// contexts are re-opened lazily and during rewriting. Not thread-safe.
+  void releaseAllDWOContexts();
 
-  /// Get Number of DWOCUs in a map.
-  uint32_t getNumDWOCUs() { return DWOCUs.size(); }
+  /// Get the number of split-DWARF CUs in the binary.
+  uint32_t getNumDWOCUs() { return DWOIdToSkeletonCU.size(); }
 
   /// Returns true if DWARF5 is used.
   bool isDWARF5Used() const { return ContainsDwarf5; }
@@ -367,7 +390,14 @@ class BinaryContext {
   std::vector<SegmentInfo> NewSegments;
 
   /// [name] -> [BinaryData*] map used for global symbol resolution.
-  using SymbolMapType = StringMap<BinaryData *>;
+  ///
+  /// The map keys are StringRefs pointing into the names owned by MCContext
+  /// (i.e. MCSymbol::getName()) rather than strings owned by this map. Every
+  /// registered name is already interned in MCContext, so keying on those
+  /// strings avoids duplicating potentially large (mangled) symbol names, which
+  /// is a significant source of memory use on large binaries. The referenced
+  /// names outlive this map, as MCContext is owned by BinaryContext.
+  using SymbolMapType = DenseMap<StringRef, BinaryData *>;
   SymbolMapType GlobalSymbols;
 
   /// [address] -> [BinaryData], ...
@@ -569,7 +599,14 @@ class BinaryContext {
   }
 
   /// Return functions meant for the output in a sorted order.
-  BinaryFunctionListType &getOutputBinaryFunctions() { return OutputFunctions; }
+  const BinaryFunctionListType &getOutputBinaryFunctions() const {
+    return OutputFunctions;
+  }
+
+  /// Update output function list.
+  void updateOutputBinaryFunctions(BinaryFunctionListType &&Functions) {
+    OutputFunctions.swap(Functions);
+  }
 
   /// Create BOLT-injected function
   BinaryFunction *createInjectedBinaryFunction(const std::string &Name,
@@ -587,6 +624,9 @@ class BinaryContext {
                          const InstructionListType &Instructions,
                          const Twine &Name = "");
 
+  /// Create a binary function with a base \p Name.
+  BinaryFunction *createThunkBinaryFunction(const std::string &Name);
+
   BinaryFunctionListType &getInjectedBinaryFunctions() {
     return InjectedBinaryFunctions;
   }

bolt/include/bolt/Core/BinaryFunction.h

@@ -54,6 +54,7 @@
 #include <algorithm>
 #include <iterator>
 #include <limits>
+#include <optional>
 #include <unordered_map>
 #include <utility>
 #include <vector>
@@ -66,6 +67,8 @@ class DWARFUnit;
 
 namespace bolt {
 
+class BranchLivenessInfo;
+
 using InputOffsetToAddressMapTy = std::unordered_multimap<uint64_t, uint64_t>;
 
 /// Types of macro-fusion alignment corrections.
@@ -403,11 +406,15 @@ class BinaryFunction {
   /// to avoid redundant processing.
   bool NeedBranchValidation{true};
 
-  /// Name for the section this function code should reside in.
-  std::string CodeSectionName;
+  /// Name for the section this function code should reside in. When unset, the
+  /// default name is derived on demand from the function's name (see
+  /// getMainSectionName()). Deferring this avoids eagerly storing a copy of the
+  /// (potentially large, mangled) function name for every function, which is a
+  /// significant source of memory use on large binaries.
+  std::optional<std::string> CodeSectionName;
 
-  /// Name for the corresponding cold code section.
-  std::string ColdCodeSectionName;
+  /// Name for the corresponding cold code section. See CodeSectionName.
+  std::optional<std::string> ColdCodeSectionName;
 
   /// Parent function fragment for split function fragments.
   using FragmentsSetTy = SmallPtrSet<BinaryFunction *, 1>;
@@ -747,12 +754,24 @@ class BinaryFunction {
   static std::string buildColdCodeSectionName(StringRef Name,
                                               const BinaryContext &BC);
 
+  /// Return the name of the main code section, using the default derived from
+  /// the function's name when no name has been explicitly assigned.
+  std::string getMainSectionName() const {
+    return CodeSectionName ? *CodeSectionName
+                           : buildCodeSectionName(getOneName(), BC);
+  }
+
+  /// Return the name of the cold code section, using the default derived from
+  /// the function's name when no name has been explicitly assigned.
+  std::string getColdSectionName() const {
+    return ColdCodeSectionName ? *ColdCodeSectionName
+                               : buildColdCodeSectionName(getOneName(), BC);
+  }
+
   /// Creation should be handled by RewriteInstance or BinaryContext
   BinaryFunction(const std::string &Name, BinarySection &Section,
                  uint64_t Address, uint64_t Size, BinaryContext &BC)
       : OriginSection(&Section), Address(Address), Size(Size), BC(BC),
-        CodeSectionName(buildCodeSectionName(Name, BC)),
-        ColdCodeSectionName(buildColdCodeSectionName(Name, BC)),
         FunctionNumber(++Count) {
     Symbols.push_back(BC.Ctx->getOrCreateSymbol(Name));
   }
@@ -1398,12 +1417,12 @@ class BinaryFunction {
   SmallString<32>
   getCodeSectionName(const FragmentNum Fragment = FragmentNum::main()) const {
     if (Fragment == FragmentNum::main())
-      return SmallString<32>(CodeSectionName);
+      return SmallString<32>(getMainSectionName());
     if (Fragment == FragmentNum::cold())
-      return SmallString<32>(ColdCodeSectionName);
+      return SmallString<32>(getColdSectionName());
     if (BC.HasWarmSection && Fragment == FragmentNum::warm())
       return SmallString<32>(BC.getWarmCodeSectionName());
-    return formatv("{0}.{1}", ColdCodeSectionName, Fragment.get() - 1);
+    return formatv("{0}.{1}", getColdSectionName(), Fragment.get() - 1);
   }
 
   /// Assign a code section name to the function.
@@ -2504,7 +2523,7 @@ class BinaryFunction {
   /// while the second successor - false/fall-through branch.
   ///
   /// When we reverse the branch condition, the CFG is updated accordingly.
-  void fixBranches();
+  void fixBranches(const BranchLivenessInfo *BLI = nullptr);
 
   /// Mark function as finalized. No further optimizations are permitted.
   void setFinalized() { CurrentState = State::CFG_Finalized; }

bolt/include/bolt/Core/BranchLivenessInfo.h

@@ -0,0 +1,43 @@
+//===- bolt/Core/BranchLivenessInfo.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_CORE_BRANCHLIVENESSINFO_H
+#define BOLT_CORE_BRANCHLIVENESSINFO_H
+
+namespace llvm {
+class MCInst;
+
+namespace bolt {
+class BinaryFunction;
+
+class BranchLivenessInfo {
+  BinaryFunction *BF;
+  unsigned AnnotationIndex;
+
+  void swap(BranchLivenessInfo &Other) noexcept;
+
+public:
+  explicit BranchLivenessInfo(BinaryFunction &BF);
+  ~BranchLivenessInfo();
+
+  // Copies would create multiple owners for removing the same annotations.
+  BranchLivenessInfo(const BranchLivenessInfo &) = delete;
+  BranchLivenessInfo &operator=(const BranchLivenessInfo &) = delete;
+
+  BranchLivenessInfo(BranchLivenessInfo &&Other) noexcept;
+  BranchLivenessInfo &operator=(BranchLivenessInfo &&Other) noexcept;
+
+  bool mustPreserveFlags(const MCInst &Inst) const;
+  void removeAnnotation(MCInst &Inst) const;
+  void setFlagsDead(MCInst &Inst);
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -475,7 +475,8 @@ class MCPlusBuilder {
   }
 
   /// Check whether this conditional branch can be reversed
-  virtual bool isReversibleBranch(const MCInst &Inst) const {
+  virtual bool isReversibleBranch(const MCInst &Inst,
+                                  bool MustPreserveFlags = true) const {
     assert(!isUnsupportedInstruction(Inst) && isConditionalBranch(Inst) &&
            "Instruction is not known conditional branch");
 
@@ -2150,9 +2151,13 @@ class MCPlusBuilder {
     llvm_unreachable("not implemented");
   }
 
-  /// Reverses the branch condition in Inst and update its taken target to TBB.
-  virtual void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
-                                      MCContext *Ctx) const {
+  /// Return the instruction sequence for the reversed branch condition of
+  /// \p Inst and update its taken target to \p TBB. Assumes that the branch is
+  /// reversible. It may replace Inst with a longer instruction sequence on some
+  /// targets.
+  virtual InstructionListType
+  reverseBranchCondition(MCInst Inst, const MCSymbol *TBB, MCContext *Ctx,
+                         bool MustPreserveFlags = true) const {
     llvm_unreachable("not implemented");
   }
 

bolt/include/bolt/Passes/BranchLivenessUtils.h

@@ -0,0 +1,28 @@
+//===- bolt/Passes/BranchLivenessUtils.h ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_BRANCHLIVENESSUTILS_H
+#define BOLT_PASSES_BRANCHLIVENESSUTILS_H
+
+#include "bolt/Core/BranchLivenessInfo.h"
+
+namespace llvm {
+namespace bolt {
+class BinaryFunction;
+class RegAnalysis;
+
+/// Return true if \p BF needs liveness info for branch transformations.
+bool needsBranchLiveness(BinaryFunction &BF);
+
+/// Return liveness info required for branch transformations.
+BranchLivenessInfo computeBranchLiveness(BinaryFunction &BF, RegAnalysis &RA);
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/Passes/LongJmp.h

@@ -14,6 +14,8 @@
 namespace llvm {
 namespace bolt {
 
+class BranchLivenessInfo;
+
 /// LongJmp is veneer-insertion pass originally written for AArch64 that
 /// compensates for its short-range branches, typically done during linking. We
 /// pull this pass inside BOLT because here we can do a better job at stub
@@ -73,8 +75,51 @@ class LongJmpPass : public BinaryFunctionPass {
 
   /// Relax all internal function branches including those between fragments.
   /// Assume that fragments are placed in different sections but are within
-  /// 128MB of each other.
-  void relaxLocalBranches(BinaryFunction &BF);
+  /// 128MB of each other. Return false and report an error if a branch cannot
+  /// be relaxed.
+  bool relaxLocalBranches(BinaryFunction &BF,
+                          const BranchLivenessInfo *BLI = nullptr);
+
+  /// A group of functions that are located within the longest direct
+  /// branch/call instruction distance. Functions within the cluster do not
+  /// require a thunk for calls in the same cluster. The cluster may include
+  /// a set of thunks for covering calls to functions outside.
+  struct FunctionCluster {
+    /// All functions in this cluster.
+    DenseSet<BinaryFunction *> Functions;
+
+    /// Symbols corresponding to entry points of functions that this cluster
+    /// calls. Note that it excludes all functions in the cluster itself.
+    DenseSet<const MCSymbol *> Callees;
+
+    /// Estimated size of the cluster in bytes.
+    uint64_t Size{0};
+
+    /// The index of the last function in the cluster. Used as an insertion
+    /// point for adding thunks to the output function list.
+    size_t LastFunctionIndex = -1;
+
+    /// When placing hot code at the end of the binary, track the first function
+    /// for insertion purposes.
+    size_t FirstFunctionIndex = -1;
+
+    /// Thunks located at the end of this cluster.
+    BinaryFunctionListType ThunkList;
+
+    /// Thunks used by this cluster. Some could be in a ThunkList of the
+    /// preceding cluster.
+    ///
+    /// <Function Symbol> -> <Thunk Function>.
+    DenseMap<const MCSymbol *, BinaryFunction *> Thunks;
+  };
+
+  /// Maximum size of combined regular functions in the cluster. Note that it's
+  /// less than 128MB, because the size of the cluster plus its thunks should be
+  /// less than 128MB.
+  static constexpr uint64_t MaxClusterSize = 125 * 1024 * 1024;
+
+  /// Relax calls using function cluster approach.
+  void relaxCalls(BinaryContext &BC);
 
   ///                 -- Layout estimation methods --
   /// Try to do layout before running the emitter, by looking at BinaryFunctions

bolt/include/bolt/Profile/Heatmap.h

@@ -76,9 +76,9 @@ class Heatmap {
 
   void print(raw_ostream &OS) const;
 
-  void printCDF(StringRef FileName) const;
+  void printCDF(StringRef FileName, StringRef Label) const;
 
-  void printCDF(raw_ostream &OS) const;
+  void printCDF(raw_ostream &OS, StringRef Label) const;
 
   void printSectionHotness(StringRef Filename) const;