diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 48f5a9609ecfb..a17910d2c1b68 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -1191,7 +1191,7 @@ void ObjFile::initSectionsAndLocalSyms(bool ignoreComdats) { if (!firstGlobal) return; - SymbolUnion *locals = makeThreadLocalN(firstGlobal); + SymbolUnion *locals = allocSymbolUnions(firstGlobal); // arena memset(locals, 0, sizeof(SymbolUnion) * firstGlobal); ArrayRef eSyms = this->getELFSyms(); diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index e4157ac53fe89..09803cc8be997 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -129,6 +129,12 @@ void elf::reportRangeError(uint8_t *loc, const Relocation &rel, const Twine &v, ", " + Twine(max).str() + "]" + hint); } +void elf::reportRelocationValueOverflow(uint64_t offset, int64_t addend) { + fatal("relocation offset (" + Twine(offset) + ") or addend (" + Twine(addend) + + ") does not fit the 16-byte packed relocation record; rebuild the linker " + "without 16-byte relocation packing"); +} + void elf::reportRangeError(uint8_t *loc, int64_t v, int n, const Symbol &sym, const Twine &msg) { ErrorPlace errPlace = getErrorPlace(loc); diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h index aaa4581490a28..19aa0bd76a745 100644 --- a/lld/ELF/Relocations.h +++ b/lld/ELF/Relocations.h @@ -120,14 +120,61 @@ enum RelExpr { R_LOONGARCH_TLSDESC_PAGE_PC, }; -// Architecture-neutral representation of relocation. +// Encode a Symbol* as a 32-bit index into the contiguous SymbolUnion arena +// (see Symbols.{h,cpp}). Every Symbol lives in that arena, so the pointer is +// symArena + index*64; index 0 is the nullptr sentinel. SymRef converts +// implicitly to/from Symbol*, so relocation `sym` use sites are unchanged. +class Symbol; +extern char *symArena; +struct SymRef { + uint32_t idx = 0; + SymRef() = default; + SymRef(Symbol *s) { *this = s; } + SymRef &operator=(Symbol *s) { + idx = s ? uint32_t((reinterpret_cast(s) - symArena) / 64) : 0; + return *this; + } + operator Symbol *() const { + return idx ? reinterpret_cast(symArena + size_t(idx) * 64) + : nullptr; + } + Symbol *operator->() const { + return reinterpret_cast(symArena + size_t(idx) * 64); + } + Symbol &operator*() const { + return *reinterpret_cast(symArena + size_t(idx) * 64); + } +}; + +// Architecture-neutral representation of relocation, packed to 16 bytes by +// additionally narrowing `offset` to uint32 and `addend` to int32 on top of the +// 32-bit `sym` index. This caps section-relative offsets at 4 GiB and addends at +// signed 32-bit, so it is a fork-local layout rather than upstreamable. The +// 16-byte size is pinned by the static_assert below. `expr` needs 7 bits and +// `type` fits in 24 bits for every supported target. The constructor moves the +// offset/addend narrowing into the mem-init list to avoid braced-init narrowing +// errors, keeping the historical (expr, type, offset, addend, sym) parameter +// order so positional initializers are unchanged. Inputs whose offset or addend +// does not fit the narrowed fields are rejected with a fatal error rather than +// silently truncated. +[[noreturn]] void reportRelocationValueOverflow(uint64_t offset, int64_t addend); struct Relocation { - RelExpr expr; - RelType type; - uint64_t offset; - int64_t addend; - Symbol *sym; + RelExpr expr : 8; + RelType type : 24; + uint32_t offset; + int32_t addend; + SymRef sym; + + Relocation() = default; + Relocation(RelExpr expr, RelType type, uint64_t offset, int64_t addend, + Symbol *sym) + : expr(expr), type(type), offset(offset), addend((int32_t)addend), + sym(sym) { + if (offset != this->offset || addend != this->addend) + reportRelocationValueOverflow(offset, addend); + } }; +static_assert(sizeof(Relocation) == 16, "Relocation should pack to 16 bytes"); // Manipulate jump instructions with these modifiers. These are used to relax // jump instruction opcodes at basic block boundaries and are particularly @@ -333,4 +380,17 @@ sortRels(Relocs> rels, bool needsGot(RelExpr expr); } // namespace lld::elf +// Let LLVM's cast<>/dyn_cast<>/isa<> see SymRef as a Symbol*, so existing +// `dyn_cast(rel.sym)` sites compile unchanged. +namespace llvm { +template <> struct simplify_type { + using SimpleType = lld::elf::Symbol *; + static SimpleType getSimplifiedValue(lld::elf::SymRef &s) { return s; } +}; +template <> struct simplify_type { + using SimpleType = lld::elf::Symbol *; + static SimpleType getSimplifiedValue(const lld::elf::SymRef &s) { return s; } +}; +} // namespace llvm + #endif diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp index 258a78ab40bb5..6ffd3269b93c0 100644 --- a/lld/ELF/SymbolTable.cpp +++ b/lld/ELF/SymbolTable.cpp @@ -85,7 +85,7 @@ Symbol *SymbolTable::insert(StringRef name) { return sym; } - Symbol *sym = reinterpret_cast(make()); + Symbol *sym = reinterpret_cast(allocSymbolUnions(1)); // arena symVector.push_back(sym); // *sym was not initialized by a constructor. Initialize all Symbol fields. diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp index 93653def328f8..c5cf130ba51d1 100644 --- a/lld/ELF/Symbols.cpp +++ b/lld/ELF/Symbols.cpp @@ -17,7 +17,10 @@ #include "lld/Common/ErrorHandler.h" #include "llvm/Demangle/Demangle.h" #include "llvm/Support/Compiler.h" +#include #include +#include +#include using namespace llvm; using namespace llvm::object; @@ -26,6 +29,34 @@ using namespace lld; using namespace lld::elf; static_assert(sizeof(SymbolUnion) <= 64, "SymbolUnion too large"); +// SymRef (Relocations.h) encodes Symbol* as index*64, so the arena stride must +// be exactly 64 for the shift-based conversion to be valid. +static_assert(sizeof(SymbolUnion) == 64, "SymRef assumes 64-byte stride"); +static_assert(alignof(SymbolUnion) <= 64, "SymbolUnion overaligned"); + +// Contiguous SymbolUnion arena. Reserve a large virtual region (only touched +// pages commit) and hand out 64-byte slots with an atomic bump. Slot 0 is +// reserved as the nullptr sentinel, so the first real symbol has index 1. +char *elf::symArena = nullptr; +namespace { +constexpr size_t symArenaCapacity = size_t(1) << 29; // 512M symbols (32 GiB VA) +std::atomic symArenaNext{1}; // index 0 == nullptr +std::once_flag symArenaOnce; +} // namespace +SymbolUnion *elf::allocSymbolUnions(size_t n) { + std::call_once(symArenaOnce, [] { + void *p = mmap(nullptr, symArenaCapacity * sizeof(SymbolUnion), + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (p == MAP_FAILED) + fatal("failed to reserve symbol arena"); + symArena = reinterpret_cast(p); + }); + size_t idx = symArenaNext.fetch_add(n, std::memory_order_relaxed); + if (idx + n > symArenaCapacity) + fatal("symbol arena exhausted"); + return reinterpret_cast(symArena) + idx; +} template struct AssertSymbol { static_assert(std::is_trivially_destructible(), diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h index e764fe8d73633..5e36bc300f38a 100644 --- a/lld/ELF/Symbols.h +++ b/lld/ELF/Symbols.h @@ -566,8 +566,15 @@ union SymbolUnion { alignas(LazySymbol) char e[sizeof(LazySymbol)]; }; +// All SymbolUnion objects are allocated from a single contiguous arena so that +// any Symbol* can be encoded as a 32-bit index (see SymRef in Relocations.h). +// `symArena` is the base; slot 0 is the nullptr sentinel, so real symbols have +// index >= 1. See Symbols.cpp for the allocator. +extern char *symArena; +SymbolUnion *allocSymbolUnions(size_t n); + template Defined *makeDefined(T &&...args) { - auto *sym = getSpecificAllocSingleton().Allocate(); + auto *sym = allocSymbolUnions(1); memset(sym, 0, sizeof(Symbol)); auto &s = *new (reinterpret_cast(sym)) Defined(std::forward(args)...); return &s;