lld: pack Relocation to 16 bytes (fork-local)

lld/ELF/InputFiles.cpp

@@ -1191,7 +1191,7 @@ void ObjFile<ELFT>::initSectionsAndLocalSyms(bool ignoreComdats) {
 
   if (!firstGlobal)
     return;
-  SymbolUnion *locals = makeThreadLocalN<SymbolUnion>(firstGlobal);
+  SymbolUnion *locals = allocSymbolUnions(firstGlobal); // arena
   memset(locals, 0, sizeof(SymbolUnion) * firstGlobal);
 
   ArrayRef<Elf_Sym> eSyms = this->getELFSyms<ELFT>();

lld/ELF/Relocations.cpp

@@ -129,6 +129,12 @@ void elf::reportRangeError(uint8_t *loc, const Relocation &rel, const Twine &v,
               ", " + Twine(max).str() + "]" + hint);
 }
 
+void elf::reportRelocationValueOverflow(uint64_t offset, int64_t addend) {
+  fatal("relocation offset (" + Twine(offset) + ") or addend (" + Twine(addend) +
+        ") does not fit the 16-byte packed relocation record; rebuild the linker "
+        "without 16-byte relocation packing");
+}
+
 void elf::reportRangeError(uint8_t *loc, int64_t v, int n, const Symbol &sym,
                            const Twine &msg) {
   ErrorPlace errPlace = getErrorPlace(loc);

lld/ELF/Relocations.h

@@ -120,14 +120,61 @@ enum RelExpr {
   R_LOONGARCH_TLSDESC_PAGE_PC,
 };
 
-// Architecture-neutral representation of relocation.
+// Encode a Symbol* as a 32-bit index into the contiguous SymbolUnion arena
+// (see Symbols.{h,cpp}). Every Symbol lives in that arena, so the pointer is
+// symArena + index*64; index 0 is the nullptr sentinel. SymRef converts
+// implicitly to/from Symbol*, so relocation `sym` use sites are unchanged.
+class Symbol;
+extern char *symArena;
+struct SymRef {
+  uint32_t idx = 0;
+  SymRef() = default;
+  SymRef(Symbol *s) { *this = s; }
+  SymRef &operator=(Symbol *s) {
+    idx = s ? uint32_t((reinterpret_cast<char *>(s) - symArena) / 64) : 0;
+    return *this;
+  }
+  operator Symbol *() const {
+    return idx ? reinterpret_cast<Symbol *>(symArena + size_t(idx) * 64)
+               : nullptr;
+  }
+  Symbol *operator->() const {
+    return reinterpret_cast<Symbol *>(symArena + size_t(idx) * 64);
+  }
+  Symbol &operator*() const {
+    return *reinterpret_cast<Symbol *>(symArena + size_t(idx) * 64);
+  }
+};
+
+// Architecture-neutral representation of relocation, packed to 16 bytes by
+// additionally narrowing `offset` to uint32 and `addend` to int32 on top of the
+// 32-bit `sym` index. This caps section-relative offsets at 4 GiB and addends at
+// signed 32-bit, so it is a fork-local layout rather than upstreamable. The
+// 16-byte size is pinned by the static_assert below. `expr` needs 7 bits and
+// `type` fits in 24 bits for every supported target. The constructor moves the
+// offset/addend narrowing into the mem-init list to avoid braced-init narrowing
+// errors, keeping the historical (expr, type, offset, addend, sym) parameter
+// order so positional initializers are unchanged. Inputs whose offset or addend
+// does not fit the narrowed fields are rejected with a fatal error rather than
+// silently truncated.
+[[noreturn]] void reportRelocationValueOverflow(uint64_t offset, int64_t addend);
 struct Relocation {
-  RelExpr expr;
-  RelType type;
-  uint64_t offset;
-  int64_t addend;
-  Symbol *sym;
+  RelExpr expr : 8;
+  RelType type : 24;
+  uint32_t offset;
+  int32_t addend;
+  SymRef sym;
+
+  Relocation() = default;
+  Relocation(RelExpr expr, RelType type, uint64_t offset, int64_t addend,
+             Symbol *sym)
+      : expr(expr), type(type), offset(offset), addend((int32_t)addend),
+        sym(sym) {
+    if (offset != this->offset || addend != this->addend)
+      reportRelocationValueOverflow(offset, addend);
+  }
 };
+static_assert(sizeof(Relocation) == 16, "Relocation should pack to 16 bytes");
 
 // Manipulate jump instructions with these modifiers.  These are used to relax
 // jump instruction opcodes at basic block boundaries and are particularly
@@ -333,4 +380,17 @@ sortRels(Relocs<llvm::object::Elf_Crel_Impl<is64>> rels,
 bool needsGot(RelExpr expr);
 } // namespace lld::elf
 
+// Let LLVM's cast<>/dyn_cast<>/isa<> see SymRef as a Symbol*, so existing
+// `dyn_cast<Defined>(rel.sym)` sites compile unchanged.
+namespace llvm {
+template <> struct simplify_type<lld::elf::SymRef> {
+  using SimpleType = lld::elf::Symbol *;
+  static SimpleType getSimplifiedValue(lld::elf::SymRef &s) { return s; }
+};
+template <> struct simplify_type<const lld::elf::SymRef> {
+  using SimpleType = lld::elf::Symbol *;
+  static SimpleType getSimplifiedValue(const lld::elf::SymRef &s) { return s; }
+};
+} // namespace llvm
+
 #endif

lld/ELF/SymbolTable.cpp

@@ -85,7 +85,7 @@ Symbol *SymbolTable::insert(StringRef name) {
     return sym;
   }
 
-  Symbol *sym = reinterpret_cast<Symbol *>(make<SymbolUnion>());
+  Symbol *sym = reinterpret_cast<Symbol *>(allocSymbolUnions(1)); // arena
   symVector.push_back(sym);
 
   // *sym was not initialized by a constructor. Initialize all Symbol fields.

lld/ELF/Symbols.cpp

@@ -17,7 +17,10 @@
 #include "lld/Common/ErrorHandler.h"
 #include "llvm/Demangle/Demangle.h"
 #include "llvm/Support/Compiler.h"
+#include <atomic>
 #include <cstring>
+#include <mutex>
+#include <sys/mman.h>
 
 using namespace llvm;
 using namespace llvm::object;
@@ -26,6 +29,34 @@ using namespace lld;
 using namespace lld::elf;
 
 static_assert(sizeof(SymbolUnion) <= 64, "SymbolUnion too large");
+// SymRef (Relocations.h) encodes Symbol* as index*64, so the arena stride must
+// be exactly 64 for the shift-based conversion to be valid.
+static_assert(sizeof(SymbolUnion) == 64, "SymRef assumes 64-byte stride");
+static_assert(alignof(SymbolUnion) <= 64, "SymbolUnion overaligned");
+
+// Contiguous SymbolUnion arena. Reserve a large virtual region (only touched
+// pages commit) and hand out 64-byte slots with an atomic bump. Slot 0 is
+// reserved as the nullptr sentinel, so the first real symbol has index 1.
+char *elf::symArena = nullptr;
+namespace {
+constexpr size_t symArenaCapacity = size_t(1) << 29; // 512M symbols (32 GiB VA)
+std::atomic<size_t> symArenaNext{1};                 // index 0 == nullptr
+std::once_flag symArenaOnce;
+} // namespace
+SymbolUnion *elf::allocSymbolUnions(size_t n) {
+  std::call_once(symArenaOnce, [] {
+    void *p = mmap(nullptr, symArenaCapacity * sizeof(SymbolUnion),
+                   PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+                   -1, 0);
+    if (p == MAP_FAILED)
+      fatal("failed to reserve symbol arena");
+    symArena = reinterpret_cast<char *>(p);
+  });
+  size_t idx = symArenaNext.fetch_add(n, std::memory_order_relaxed);
+  if (idx + n > symArenaCapacity)
+    fatal("symbol arena exhausted");
+  return reinterpret_cast<SymbolUnion *>(symArena) + idx;
+}
 
 template <typename T> struct AssertSymbol {
   static_assert(std::is_trivially_destructible<T>(),

lld/ELF/Symbols.h

@@ -566,8 +566,15 @@ union SymbolUnion {
   alignas(LazySymbol) char e[sizeof(LazySymbol)];
 };
 
+// All SymbolUnion objects are allocated from a single contiguous arena so that
+// any Symbol* can be encoded as a 32-bit index (see SymRef in Relocations.h).
+// `symArena` is the base; slot 0 is the nullptr sentinel, so real symbols have
+// index >= 1. See Symbols.cpp for the allocator.
+extern char *symArena;
+SymbolUnion *allocSymbolUnions(size_t n);
+
 template <typename... T> Defined *makeDefined(T &&...args) {
-  auto *sym = getSpecificAllocSingleton<SymbolUnion>().Allocate();
+  auto *sym = allocSymbolUnions(1);
   memset(sym, 0, sizeof(Symbol));
   auto &s = *new (reinterpret_cast<Defined *>(sym)) Defined(std::forward<T>(args)...);
   return &s;