From 02fcab7af8c04d91aa38009afdd815137650f6f5 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 1 Oct 2026 13:23:32 -0400 Subject: [PATCH 1/4] Store a sent packet's first stream send inline Every packet that carries stream data records which stream sends it carried in `TransmittedItems.sentStreams`, and keeps the record until the packet is acknowledged or declared lost. The record was an array, so each such packet allocated storage for it, though a packet almost always carries one stream. `SentStreams` keeps the first send inline and only allocates for the rest. `SentStream` holds nothing but values, so it is now copyable, which lets the inline slot be an ordinary optional. Measured on my Mac against `main` with the package's QUIC benchmark tools. Allocation counts come from full malloc stack logging, which records every allocation: QUICTransfer -size 1200, per message 24.37 -> 23.31 QUICTransfer, per 500 KB transfer 6,187.6 -> 5,747.9 QUICHandshake, per connection 1,947.4 -> 1,944.8 QUICStreamLoad, per stream 111.8 -> 109.8 The stream load still allocates for a second send in about one packet in seventeen. Wall-clock time comes from running `main`, this change and the other changes measured alongside it in a rotating order for 9 rounds, and comparing each run with `main`'s in the same round. Changes moved paths they do not touch by up to about 1.3%, so differences that size count as noise. Nothing changed beyond noise. --- Sources/SwiftNetwork/QUIC/SendItems.swift | 38 +++++++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/Sources/SwiftNetwork/QUIC/SendItems.swift b/Sources/SwiftNetwork/QUIC/SendItems.swift index 3f2017b2..6a5c2263 100644 --- a/Sources/SwiftNetwork/QUIC/SendItems.swift +++ b/Sources/SwiftNetwork/QUIC/SendItems.swift @@ -3110,19 +3110,51 @@ struct TransmittedItems: ~Copyable { var sentCrypto = NetworkUniqueArray() // Minimal information about a send on a stream - struct SentStream: ~Copyable { + struct SentStream { let flowID: MultiplexedFlowIdentifier let streamID: QUICStreamID let offset: UInt64 let length: UInt64 let isFinal: Bool - func matches(_ other: borrowing SentStream) -> Bool { + func matches(_ other: SentStream) -> Bool { flowID == other.flowID && streamID == other.streamID && offset == other.offset && length == other.length && isFinal == other.isFinal } } - var sentStreams = NetworkUniqueArray() + + /// The stream sends a packet carried. + /// + /// A packet almost always carries a single stream, and every sent packet keeps its record until it is + /// acknowledged or lost, so the first send is stored inline and only the rest need storage of their own. + struct SentStreams: ~Copyable { + private var first: SentStream? + private var rest = NetworkUniqueArray() + + var isEmpty: Bool { + first == nil + } + + var count: Int { + first == nil ? 0 : 1 + rest.count + } + + subscript(index: Int) -> SentStream { + if index == 0, let first { + return first + } + return rest[index - 1] + } + + mutating func append(_ sentStream: SentStream) { + if first == nil { + first = sentStream + } else { + rest.append(sentStream) + } + } + } + var sentStreams = SentStreams() var maxStreamDataFlows = Deque() var streamDataBlockedFlows = Deque() From d86102fc9077b68ad1df4552ca1b4920c4a4b405 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Fri, 2 Oct 2026 11:47:08 -0400 Subject: [PATCH 2/4] Add `NetworkSmallUniqueArray` An array of potentially noncopyable elements that keeps a fixed number of them inline and spills to heap storage only past that, so the common case of one or two elements allocates nothing. Taken unchanged from apple/swift-network-evolution#131. Co-authored-by: Matt Eaton --- .../Utilities/NetworkSmallUniqueArray.swift | 430 ++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift diff --git a/Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift b/Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift new file mode 100644 index 00000000..a0ce81e2 --- /dev/null +++ b/Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift @@ -0,0 +1,430 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(BasicContainers) + +import BasicContainers + +/// A trivial blob of memory used as raw inline storage for one element. +/// +/// Conformers must be bitwise-copyable and have no meaningful value of their +/// own: ``NetworkSmallUniqueArray`` reinterprets them as element storage. +@available(Network 0.1.0, *) +protocol NetworkInlineStorageSlot: BitwiseCopyable, Sendable { + /// An all-zero instance, used to bring the raw storage into existence. + static var zero: Self { get } +} + +@available(Network 0.1.0, *) +extension InlineArray: NetworkInlineStorageSlot where Element == UInt64 { + @usableFromInline + static var zero: Self { .init(repeating: 0) } +} + +/// An element type that ``NetworkSmallUniqueArray`` can store inline. +/// +/// Conformers declare a slot large enough to hold one element. Stating it once +/// here, next to the element type, is what lets `NetworkSmallUniqueArray` take +/// its capacity as a count of *elements* rather than a count of machine words: +/// +/// ```swift +/// extension Frame: NetworkInlineStorable { +/// // Frame is 136 bytes. +/// typealias InlineSlot = InlineArray<17, UInt64> +/// } +/// +/// var frames = NetworkSmallUniqueArray() +/// ``` +/// +/// Swift's value generics cannot yet perform arithmetic or read `MemoryLayout`, +/// so the slot's word count has to be written out. It only has to be written +/// once, and ``NetworkSmallUniqueArray`` checks it on construction, so growing +/// the element type past its slot traps with a clear message instead of +/// corrupting memory. A test asserting the element's `stride` will catch it even +/// earlier. +@available(Network 0.1.0, *) +protocol NetworkInlineStorable: ~Copyable { + /// Raw storage at least `MemoryLayout.stride` bytes wide, and at least + /// as aligned as `Self`. + associatedtype InlineSlot: NetworkInlineStorageSlot +} + +/// An array of potentially noncopyable elements that stores a small number of +/// elements inline and spills to heap storage only when it has to. +/// +/// This is tuned for the shape of traffic seen by `FrameArray` and the +/// `PendingEvent` queues: almost always one or two elements, occasionally many. +/// The design goal is that the common cases cost as close to nothing as +/// possible. +/// +/// ## Representation +/// +/// The inline storage is a *trivial* `InlineArray` of `Element.InlineSlot`, and +/// `_count` alone records which slots hold live elements. Two properties follow +/// from that, and they are the whole reason this type is fast: +/// +/// - **Creating an empty array writes nothing.** There are no per-slot +/// `Optional` tags to set to `nil`. Because nothing reads the raw storage +/// until `_count` says it is live, the optimizer deletes the storage +/// initialization outright. +/// - **No per-element tag traffic.** Appending writes only the element and the +/// count; removing reads only the count. A tagged representation must also +/// write a discriminator on every append and test it on every access and on +/// every deinit. +/// +/// Two tagged alternatives were measured against a 136-byte `Frame` and both are +/// dramatically worse, for the same underlying reason — the tag is load-bearing +/// on the destroy path, so it cannot be optimized away: +/// +/// | Inline storage | append + remove | empty array | +/// | --- | --- | --- | +/// | `InlineArray` (this type) | 1.0 ns/op | 18 instr | +/// | `InlineArray` | 28.0 ns/op | 56 instr | +/// | `Optional>` | 12.1 ns/op | 19 instr | +/// +/// The third row is worth calling out because it looks ideal: the compiler sizes +/// the storage exactly, so no slot declaration is needed at all. It is 12× slower. +/// +/// Elements beyond `InlineCapacity` spill into a `NetworkUniqueArray`, which +/// performs no allocation while it is empty, so the overflow path costs nothing +/// until it is used. +/// +/// ## Capacity +/// +/// `InlineCapacity` is a number of elements. Size it for the number you actually +/// expect to be common: inline storage is part of the containing value, so a +/// large capacity times a large `Element` makes every move of the enclosing type +/// more expensive. +@available(Network 0.1.0, *) +struct NetworkSmallUniqueArray< + Element: NetworkInlineStorable & ~Copyable, + let InlineCapacity: Int +>: ~Copyable { + /// Trivial raw backing store for the inline elements. + /// + /// Only the first `min(_count, InlineCapacity)` element-strides hold + /// initialized elements; the rest is uninitialized garbage. + var _inlineStorage: InlineArray + + /// The total number of elements, inline plus overflow. + /// + /// This doubles as the tag: it is the only thing that says which inline + /// slots are live. + var _count: Int + + /// Elements past `InlineCapacity`. Allocates nothing while empty. + var _overflowStorage: NetworkUniqueArray + + /// The number of elements that fit inline. + @inline(always) + static var inlineCapacity: Int { InlineCapacity } + + /// Traps if `Element.InlineSlot` cannot hold an `Element`. + /// + /// This is the one invariant a slot declaration has to satisfy. Swift has no + /// static assertion, so it is checked on construction instead; the check + /// folds away entirely when it holds. + @inline(always) + static func _checkInlineSlot() { + precondition( + MemoryLayout.stride >= MemoryLayout.stride, + "InlineSlot is too small to hold Element" + ) + precondition( + MemoryLayout.alignment >= MemoryLayout.alignment, + "InlineSlot is less aligned than Element" + ) + } + + @inline(always) + init() { + Self._checkInlineSlot() + // Never read before `_count` marks a slot live, so this initialization + // is dead and the optimizer removes it. + _inlineStorage = .init({ _ in .zero }) + _count = 0 + _overflowStorage = .init(minimumCapacity: 0) + } + + /// The total number of elements. + @inline(always) + var count: Int { _count } + + @inline(always) + var isEmpty: Bool { _count == 0 } + + /// The number of elements currently held in inline storage. + @inline(always) + var _inlineCount: Int { + Swift.min(_count, InlineCapacity) + } + + // MARK: - Adding elements + + /// Appends an element, spilling to heap storage past the inline capacity. + @inline(always) + mutating func append(_ element: consuming Element) { + if _fastPath(_count < InlineCapacity) { + let index = _count + // An Optional shuttle is how a consumed value crosses into the + // pointer store; a closure cannot consume a capture. + var shuttle: Element? = consume element + _inlineBase().advanced(by: index).initialize(to: shuttle.take()!) + } else { + _appendToOverflow(element) + } + _count &+= 1 + } + + /// Out-of-line slow path, kept separate so it does not bloat `append`. + @inline(never) + mutating func _appendToOverflow(_ element: consuming Element) { + _overflowStorage.append(element) + } + + /// Reserves enough overflow storage to hold `n` elements beyond + /// `InlineCapacity` without reallocating. + /// + /// Inline capacity is fixed at compile time, so this only affects the + /// overflow storage. + @inline(always) + mutating func reserveCapacity(_ n: Int) { + let overflowCount = max(0, n &- InlineCapacity) + _overflowStorage.reserveCapacity(overflowCount) + } + + // MARK: - Removing elements + + /// Removes and returns the first element. + /// + /// - Precondition: The array is not empty. + @inline(always) + mutating func removeFirst() -> Element { + precondition(_count > 0, "Can't remove first element from an empty array") + return _removeInline(at: 0) + } + + /// Removes and returns the element at `index`. + /// + /// - Precondition: `index` is a valid index of the array. + @discardableResult + mutating func remove(at index: Int) -> Element { + precondition(index >= 0 && index < _count, "Index out of range") + if index >= InlineCapacity { + // Entirely within overflow storage; nothing inline has to move. + let removed = _overflowStorage.remove(at: index &- InlineCapacity) + _count &-= 1 + return removed + } + return _removeInline(at: index) + } + + /// Removes an element held in inline storage, closing the gap it leaves. + @inline(always) + mutating func _removeInline(at index: Int) -> Element { + let inlineCount = _inlineCount + let base = _inlineBase() + let removed = base.advanced(by: index).move() + // Slide the surviving inline elements down over the hole. + var source = index &+ 1 + while source < inlineCount { + base.advanced(by: source &- 1).initialize(to: base.advanced(by: source).move()) + source &+= 1 + } + // Promote the first overflow element into the slot that just opened up, + // keeping inline storage densely packed. + if _slowPath(!_overflowStorage.isEmpty) { + _promoteFirstOverflowElement(to: inlineCount &- 1) + } + _count &-= 1 + return removed + } + + /// Moves the first overflow element into the given inline slot. + @inline(never) + mutating func _promoteFirstOverflowElement(to index: Int) { + var shuttle: Element? = _overflowStorage.remove(at: 0) + _inlineBase().advanced(by: index).initialize(to: shuttle.take()!) + } + + // MARK: - Accessing elements + + /// Calls `body` with the element at `index`, borrowed in place. + /// + /// Elements are reached through scoped closures rather than a subscript + /// because a subscript's read accessor would have to derive an address from + /// a *borrow* of the inline storage, and such an address does not reliably + /// point at the real storage. + /// + /// - Precondition: `index` is a valid index of the array. + @inline(always) + mutating func withElement( + at index: Int, + _ body: (borrowing Element) -> R + ) -> R { + precondition(index >= 0 && index < _count, "Index out of range") + if index >= InlineCapacity { + return body(_overflowStorage[index &- InlineCapacity]) + } + return body(_inlineBase().advanced(by: index).pointee) + } + + /// Calls `body` with the element at `index`, available for mutation in + /// place. The element is never copied or moved. + /// + /// - Precondition: `index` is a valid index of the array. + @inline(always) + mutating func withMutableElement( + at index: Int, + _ body: (inout Element) -> R + ) -> R { + precondition(index >= 0 && index < _count, "Index out of range") + if index >= InlineCapacity { + return body(&_overflowStorage[index &- InlineCapacity]) + } + return body(&_inlineBase().advanced(by: index).pointee) + } + + // MARK: - Raw storage addressing + + /// The address of inline element 0. + /// + /// The address has to come from the storage's `mutableSpan`: + /// `withUnsafeMutablePointer(to: &_inlineStorage)` may hand back a pointer + /// into a *temporary copy* of the trivial storage, so writes through it are + /// silently lost once the array also holds overflow elements. + /// + /// Extracting the pointer into a typed `let` and returning it — rather than + /// doing the work inside `withUnsafeMutableBufferPointer` — is what keeps + /// this allocation-free. Performing the element store inside that closure + /// makes the `Optional` shuttle escape into a heap box, which measured four + /// `swift_slowDealloc` calls and 132 instructions on the hot path versus 27 + /// here. + @inline(always) + mutating func _inlineBase() -> UnsafeMutablePointer { + var span = _inlineStorage.mutableSpan + let base: UnsafeMutablePointer = span.withUnsafeMutableBufferPointer { buffer in + UnsafeMutableRawPointer(buffer.baseAddress.unsafelyUnwrapped) + .assumingMemoryBound(to: Element.self) + } + return base + } + + deinit { + let inlineCount = _inlineCount + if inlineCount > 0 { + // `span` is unavailable here (it would escape a borrow of a value + // being destroyed), so address the storage directly. Unlike the + // mutating paths this only has to *read* the elements out to + // destroy them, and `withUnsafePointer` guarantees the copy it may + // make holds the same element values. + let _ = withUnsafePointer(to: _inlineStorage) { storage in + UnsafeMutableRawPointer(mutating: UnsafeRawPointer(storage)) + .assumingMemoryBound(to: Element.self) + .deinitialize(count: inlineCount) + } + } + // `_overflowStorage` destroys its own elements. + } +} + +// MARK: - Copyable element convenience + +@available(Network 0.1.0, *) +extension NetworkSmallUniqueArray where Element: Copyable { + /// Reads the element at `index` by copy, without requiring mutating access. + /// + /// - Precondition: `index` is a valid index of the array. + @inline(always) + func _copyElement(at index: Int) -> Element { + precondition(index >= 0 && index < _count, "Index out of range") + if index >= InlineCapacity { + return _overflowStorage[index &- InlineCapacity] + } + return withUnsafePointer(to: _inlineStorage) { storage in + UnsafeRawPointer(storage) + .assumingMemoryBound(to: Element.self)[index] + } + } + + /// Accesses the element at `index` by copy. + /// + /// Only available for `Copyable` elements: a subscript's read accessor + /// would otherwise have to derive an address from a borrow of the inline + /// storage, which does not reliably point at the real storage. See + /// ``withElement(at:_:)`` and ``withMutableElement(at:_:)`` for the + /// noncopyable-safe access pattern this bypasses. + @inline(always) + subscript(index: Int) -> Element { + get { _copyElement(at: index) } + set { withMutableElement(at: index) { $0 = newValue } } + } + + /// Creates an array holding the elements of `sequence`, in order. + @inline(always) + init(_ sequence: S) where S.Element == Element { + self.init() + for element in sequence { + append(element) + } + } + + /// Creates an array with `count` copies of `element`. + @inline(always) + init(repeating element: Element, count: Int) { + self.init() + for _ in 0.. [Element] { + var result = [Element]() + result.reserveCapacity(_count) + let inlineCount = _inlineCount + if inlineCount > 0 { + withUnsafePointer(to: _inlineStorage) { storage in + let base = UnsafeRawPointer(storage).assumingMemoryBound(to: Element.self) + for i in 0.. +} + +/// `ProtocolEventManagerState.PendingEvent` is 328 bytes (326 rounded to stride). +@available(Network 0.1.0, *) +extension ProtocolEventManagerState.PendingEvent: NetworkInlineStorable { + typealias InlineSlot = InlineArray<41, UInt64> +} + +#endif From e37fbc73aea367de2bae33e1692d1b2246cd34a2 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Fri, 2 Oct 2026 11:50:29 -0400 Subject: [PATCH 3/4] Store a sent packet's stream sends in a `NetworkSmallUniqueArray` `SentStreams` was a one-element inline array written for this one use. `NetworkSmallUniqueArray` does the same job, so the hand-written type goes and `SentStream` declares its inline slot instead. The layout test still expected the size `SentPacketRecord` had before its first stream send was stored inline. Holding that send, and the count the small array keeps, takes the record from 193 bytes to 241; the test now expects that, in exchange for the allocation each record used to make. --- Sources/SwiftNetwork/QUIC/SendItems.swift | 39 ++++------------------- Tests/QUICTests/QUICLayoutTests.swift | 2 +- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/Sources/SwiftNetwork/QUIC/SendItems.swift b/Sources/SwiftNetwork/QUIC/SendItems.swift index 6a5c2263..d39780c6 100644 --- a/Sources/SwiftNetwork/QUIC/SendItems.swift +++ b/Sources/SwiftNetwork/QUIC/SendItems.swift @@ -3123,38 +3123,8 @@ struct TransmittedItems: ~Copyable { } } - /// The stream sends a packet carried. - /// - /// A packet almost always carries a single stream, and every sent packet keeps its record until it is - /// acknowledged or lost, so the first send is stored inline and only the rest need storage of their own. - struct SentStreams: ~Copyable { - private var first: SentStream? - private var rest = NetworkUniqueArray() - - var isEmpty: Bool { - first == nil - } - - var count: Int { - first == nil ? 0 : 1 + rest.count - } - - subscript(index: Int) -> SentStream { - if index == 0, let first { - return first - } - return rest[index - 1] - } - - mutating func append(_ sentStream: SentStream) { - if first == nil { - first = sentStream - } else { - rest.append(sentStream) - } - } - } - var sentStreams = SentStreams() + /// The stream sends a packet carried. A packet almost always carries one, so it is stored inline. + var sentStreams = NetworkSmallUniqueArray() var maxStreamDataFlows = Deque() var streamDataBlockedFlows = Deque() @@ -3248,4 +3218,9 @@ struct TransmittedItems: ~Copyable { } } +/// `SentStream` is 33 bytes, 40 to its stride. +@available(Network 0.1.0, *) +extension TransmittedItems.SentStream: NetworkInlineStorable { + typealias InlineSlot = InlineArray<5, UInt64> +} #endif diff --git a/Tests/QUICTests/QUICLayoutTests.swift b/Tests/QUICTests/QUICLayoutTests.swift index a33a3144..4ac40462 100644 --- a/Tests/QUICTests/QUICLayoutTests.swift +++ b/Tests/QUICTests/QUICLayoutTests.swift @@ -31,7 +31,7 @@ final class QUICLayoutTests: XCTestCase { func testLayoutPacket() { let packetSize = 156 - let packetRecordSize = 193 + let packetRecordSize = 241 XCTAssertEqual(packetSize, MemoryLayout.size) XCTAssertEqual(packetRecordSize, MemoryLayout.size) } From 5e991ad8a0c7d4e4a00768beeadabccffff71df8 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Fri, 2 Oct 2026 14:09:12 -0400 Subject: [PATCH 4/4] Hold a sent packet's first stream send in an enum `SentStreams` keeps its sends in an enum: none, one inline, or an array once a second send arrives. That covers the case that matters, a packet carrying one stream, without the raw inline storage and per-element slot declarations `NetworkSmallUniqueArray` needs, so that type goes. `SentStream` is plain values, so the record holds the single send directly, and `SentPacketRecord` is 209 bytes rather than 241; the layout test now expects that. --- Sources/SwiftNetwork/QUIC/SendItems.swift | 66 ++- .../Utilities/NetworkSmallUniqueArray.swift | 430 ------------------ Tests/QUICTests/QUICLayoutTests.swift | 2 +- 3 files changed, 60 insertions(+), 438 deletions(-) delete mode 100644 Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift diff --git a/Sources/SwiftNetwork/QUIC/SendItems.swift b/Sources/SwiftNetwork/QUIC/SendItems.swift index d39780c6..54da19f5 100644 --- a/Sources/SwiftNetwork/QUIC/SendItems.swift +++ b/Sources/SwiftNetwork/QUIC/SendItems.swift @@ -3123,8 +3123,65 @@ struct TransmittedItems: ~Copyable { } } - /// The stream sends a packet carried. A packet almost always carries one, so it is stored inline. - var sentStreams = NetworkSmallUniqueArray() + /// The stream sends a packet carried. + /// + /// A packet almost always carries a single stream, and every sent packet keeps its record until it is + /// acknowledged or lost, so one send is held inline and only a second one allocates storage. + struct SentStreams: ~Copyable { + private enum Storage: ~Copyable { + case empty + case one(SentStream) + case many(NetworkUniqueArray) + } + + private var storage = Storage.empty + + var isEmpty: Bool { + switch storage { + case .empty: return true + case .one: return false + case .many(let sends): return sends.isEmpty + } + } + + var count: Int { + switch storage { + case .empty: return 0 + case .one: return 1 + case .many(let sends): return sends.count + } + } + + subscript(index: Int) -> SentStream { + switch storage { + case .empty: + preconditionFailure("Index out of range") + case .one(let send): + precondition(index == 0, "Index out of range") + return send + case .many(let sends): + return sends[index] + } + } + + mutating func append(_ sentStream: SentStream) { + var taken = Storage.empty + swap(&taken, &storage) + switch consume taken { + case .empty: + storage = .one(sentStream) + case .one(let first): + var sends = NetworkUniqueArray(minimumCapacity: 2) + sends.append(first) + sends.append(sentStream) + storage = .many(sends) + case .many(var sends): + sends.append(sentStream) + storage = .many(sends) + } + } + } + var sentStreams = SentStreams() var maxStreamDataFlows = Deque() var streamDataBlockedFlows = Deque() @@ -3218,9 +3275,4 @@ struct TransmittedItems: ~Copyable { } } -/// `SentStream` is 33 bytes, 40 to its stride. -@available(Network 0.1.0, *) -extension TransmittedItems.SentStream: NetworkInlineStorable { - typealias InlineSlot = InlineArray<5, UInt64> -} #endif diff --git a/Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift b/Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift deleted file mode 100644 index a0ce81e2..00000000 --- a/Sources/SwiftNetwork/Utilities/NetworkSmallUniqueArray.swift +++ /dev/null @@ -1,430 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2026 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#if canImport(BasicContainers) - -import BasicContainers - -/// A trivial blob of memory used as raw inline storage for one element. -/// -/// Conformers must be bitwise-copyable and have no meaningful value of their -/// own: ``NetworkSmallUniqueArray`` reinterprets them as element storage. -@available(Network 0.1.0, *) -protocol NetworkInlineStorageSlot: BitwiseCopyable, Sendable { - /// An all-zero instance, used to bring the raw storage into existence. - static var zero: Self { get } -} - -@available(Network 0.1.0, *) -extension InlineArray: NetworkInlineStorageSlot where Element == UInt64 { - @usableFromInline - static var zero: Self { .init(repeating: 0) } -} - -/// An element type that ``NetworkSmallUniqueArray`` can store inline. -/// -/// Conformers declare a slot large enough to hold one element. Stating it once -/// here, next to the element type, is what lets `NetworkSmallUniqueArray` take -/// its capacity as a count of *elements* rather than a count of machine words: -/// -/// ```swift -/// extension Frame: NetworkInlineStorable { -/// // Frame is 136 bytes. -/// typealias InlineSlot = InlineArray<17, UInt64> -/// } -/// -/// var frames = NetworkSmallUniqueArray() -/// ``` -/// -/// Swift's value generics cannot yet perform arithmetic or read `MemoryLayout`, -/// so the slot's word count has to be written out. It only has to be written -/// once, and ``NetworkSmallUniqueArray`` checks it on construction, so growing -/// the element type past its slot traps with a clear message instead of -/// corrupting memory. A test asserting the element's `stride` will catch it even -/// earlier. -@available(Network 0.1.0, *) -protocol NetworkInlineStorable: ~Copyable { - /// Raw storage at least `MemoryLayout.stride` bytes wide, and at least - /// as aligned as `Self`. - associatedtype InlineSlot: NetworkInlineStorageSlot -} - -/// An array of potentially noncopyable elements that stores a small number of -/// elements inline and spills to heap storage only when it has to. -/// -/// This is tuned for the shape of traffic seen by `FrameArray` and the -/// `PendingEvent` queues: almost always one or two elements, occasionally many. -/// The design goal is that the common cases cost as close to nothing as -/// possible. -/// -/// ## Representation -/// -/// The inline storage is a *trivial* `InlineArray` of `Element.InlineSlot`, and -/// `_count` alone records which slots hold live elements. Two properties follow -/// from that, and they are the whole reason this type is fast: -/// -/// - **Creating an empty array writes nothing.** There are no per-slot -/// `Optional` tags to set to `nil`. Because nothing reads the raw storage -/// until `_count` says it is live, the optimizer deletes the storage -/// initialization outright. -/// - **No per-element tag traffic.** Appending writes only the element and the -/// count; removing reads only the count. A tagged representation must also -/// write a discriminator on every append and test it on every access and on -/// every deinit. -/// -/// Two tagged alternatives were measured against a 136-byte `Frame` and both are -/// dramatically worse, for the same underlying reason — the tag is load-bearing -/// on the destroy path, so it cannot be optimized away: -/// -/// | Inline storage | append + remove | empty array | -/// | --- | --- | --- | -/// | `InlineArray` (this type) | 1.0 ns/op | 18 instr | -/// | `InlineArray` | 28.0 ns/op | 56 instr | -/// | `Optional>` | 12.1 ns/op | 19 instr | -/// -/// The third row is worth calling out because it looks ideal: the compiler sizes -/// the storage exactly, so no slot declaration is needed at all. It is 12× slower. -/// -/// Elements beyond `InlineCapacity` spill into a `NetworkUniqueArray`, which -/// performs no allocation while it is empty, so the overflow path costs nothing -/// until it is used. -/// -/// ## Capacity -/// -/// `InlineCapacity` is a number of elements. Size it for the number you actually -/// expect to be common: inline storage is part of the containing value, so a -/// large capacity times a large `Element` makes every move of the enclosing type -/// more expensive. -@available(Network 0.1.0, *) -struct NetworkSmallUniqueArray< - Element: NetworkInlineStorable & ~Copyable, - let InlineCapacity: Int ->: ~Copyable { - /// Trivial raw backing store for the inline elements. - /// - /// Only the first `min(_count, InlineCapacity)` element-strides hold - /// initialized elements; the rest is uninitialized garbage. - var _inlineStorage: InlineArray - - /// The total number of elements, inline plus overflow. - /// - /// This doubles as the tag: it is the only thing that says which inline - /// slots are live. - var _count: Int - - /// Elements past `InlineCapacity`. Allocates nothing while empty. - var _overflowStorage: NetworkUniqueArray - - /// The number of elements that fit inline. - @inline(always) - static var inlineCapacity: Int { InlineCapacity } - - /// Traps if `Element.InlineSlot` cannot hold an `Element`. - /// - /// This is the one invariant a slot declaration has to satisfy. Swift has no - /// static assertion, so it is checked on construction instead; the check - /// folds away entirely when it holds. - @inline(always) - static func _checkInlineSlot() { - precondition( - MemoryLayout.stride >= MemoryLayout.stride, - "InlineSlot is too small to hold Element" - ) - precondition( - MemoryLayout.alignment >= MemoryLayout.alignment, - "InlineSlot is less aligned than Element" - ) - } - - @inline(always) - init() { - Self._checkInlineSlot() - // Never read before `_count` marks a slot live, so this initialization - // is dead and the optimizer removes it. - _inlineStorage = .init({ _ in .zero }) - _count = 0 - _overflowStorage = .init(minimumCapacity: 0) - } - - /// The total number of elements. - @inline(always) - var count: Int { _count } - - @inline(always) - var isEmpty: Bool { _count == 0 } - - /// The number of elements currently held in inline storage. - @inline(always) - var _inlineCount: Int { - Swift.min(_count, InlineCapacity) - } - - // MARK: - Adding elements - - /// Appends an element, spilling to heap storage past the inline capacity. - @inline(always) - mutating func append(_ element: consuming Element) { - if _fastPath(_count < InlineCapacity) { - let index = _count - // An Optional shuttle is how a consumed value crosses into the - // pointer store; a closure cannot consume a capture. - var shuttle: Element? = consume element - _inlineBase().advanced(by: index).initialize(to: shuttle.take()!) - } else { - _appendToOverflow(element) - } - _count &+= 1 - } - - /// Out-of-line slow path, kept separate so it does not bloat `append`. - @inline(never) - mutating func _appendToOverflow(_ element: consuming Element) { - _overflowStorage.append(element) - } - - /// Reserves enough overflow storage to hold `n` elements beyond - /// `InlineCapacity` without reallocating. - /// - /// Inline capacity is fixed at compile time, so this only affects the - /// overflow storage. - @inline(always) - mutating func reserveCapacity(_ n: Int) { - let overflowCount = max(0, n &- InlineCapacity) - _overflowStorage.reserveCapacity(overflowCount) - } - - // MARK: - Removing elements - - /// Removes and returns the first element. - /// - /// - Precondition: The array is not empty. - @inline(always) - mutating func removeFirst() -> Element { - precondition(_count > 0, "Can't remove first element from an empty array") - return _removeInline(at: 0) - } - - /// Removes and returns the element at `index`. - /// - /// - Precondition: `index` is a valid index of the array. - @discardableResult - mutating func remove(at index: Int) -> Element { - precondition(index >= 0 && index < _count, "Index out of range") - if index >= InlineCapacity { - // Entirely within overflow storage; nothing inline has to move. - let removed = _overflowStorage.remove(at: index &- InlineCapacity) - _count &-= 1 - return removed - } - return _removeInline(at: index) - } - - /// Removes an element held in inline storage, closing the gap it leaves. - @inline(always) - mutating func _removeInline(at index: Int) -> Element { - let inlineCount = _inlineCount - let base = _inlineBase() - let removed = base.advanced(by: index).move() - // Slide the surviving inline elements down over the hole. - var source = index &+ 1 - while source < inlineCount { - base.advanced(by: source &- 1).initialize(to: base.advanced(by: source).move()) - source &+= 1 - } - // Promote the first overflow element into the slot that just opened up, - // keeping inline storage densely packed. - if _slowPath(!_overflowStorage.isEmpty) { - _promoteFirstOverflowElement(to: inlineCount &- 1) - } - _count &-= 1 - return removed - } - - /// Moves the first overflow element into the given inline slot. - @inline(never) - mutating func _promoteFirstOverflowElement(to index: Int) { - var shuttle: Element? = _overflowStorage.remove(at: 0) - _inlineBase().advanced(by: index).initialize(to: shuttle.take()!) - } - - // MARK: - Accessing elements - - /// Calls `body` with the element at `index`, borrowed in place. - /// - /// Elements are reached through scoped closures rather than a subscript - /// because a subscript's read accessor would have to derive an address from - /// a *borrow* of the inline storage, and such an address does not reliably - /// point at the real storage. - /// - /// - Precondition: `index` is a valid index of the array. - @inline(always) - mutating func withElement( - at index: Int, - _ body: (borrowing Element) -> R - ) -> R { - precondition(index >= 0 && index < _count, "Index out of range") - if index >= InlineCapacity { - return body(_overflowStorage[index &- InlineCapacity]) - } - return body(_inlineBase().advanced(by: index).pointee) - } - - /// Calls `body` with the element at `index`, available for mutation in - /// place. The element is never copied or moved. - /// - /// - Precondition: `index` is a valid index of the array. - @inline(always) - mutating func withMutableElement( - at index: Int, - _ body: (inout Element) -> R - ) -> R { - precondition(index >= 0 && index < _count, "Index out of range") - if index >= InlineCapacity { - return body(&_overflowStorage[index &- InlineCapacity]) - } - return body(&_inlineBase().advanced(by: index).pointee) - } - - // MARK: - Raw storage addressing - - /// The address of inline element 0. - /// - /// The address has to come from the storage's `mutableSpan`: - /// `withUnsafeMutablePointer(to: &_inlineStorage)` may hand back a pointer - /// into a *temporary copy* of the trivial storage, so writes through it are - /// silently lost once the array also holds overflow elements. - /// - /// Extracting the pointer into a typed `let` and returning it — rather than - /// doing the work inside `withUnsafeMutableBufferPointer` — is what keeps - /// this allocation-free. Performing the element store inside that closure - /// makes the `Optional` shuttle escape into a heap box, which measured four - /// `swift_slowDealloc` calls and 132 instructions on the hot path versus 27 - /// here. - @inline(always) - mutating func _inlineBase() -> UnsafeMutablePointer { - var span = _inlineStorage.mutableSpan - let base: UnsafeMutablePointer = span.withUnsafeMutableBufferPointer { buffer in - UnsafeMutableRawPointer(buffer.baseAddress.unsafelyUnwrapped) - .assumingMemoryBound(to: Element.self) - } - return base - } - - deinit { - let inlineCount = _inlineCount - if inlineCount > 0 { - // `span` is unavailable here (it would escape a borrow of a value - // being destroyed), so address the storage directly. Unlike the - // mutating paths this only has to *read* the elements out to - // destroy them, and `withUnsafePointer` guarantees the copy it may - // make holds the same element values. - let _ = withUnsafePointer(to: _inlineStorage) { storage in - UnsafeMutableRawPointer(mutating: UnsafeRawPointer(storage)) - .assumingMemoryBound(to: Element.self) - .deinitialize(count: inlineCount) - } - } - // `_overflowStorage` destroys its own elements. - } -} - -// MARK: - Copyable element convenience - -@available(Network 0.1.0, *) -extension NetworkSmallUniqueArray where Element: Copyable { - /// Reads the element at `index` by copy, without requiring mutating access. - /// - /// - Precondition: `index` is a valid index of the array. - @inline(always) - func _copyElement(at index: Int) -> Element { - precondition(index >= 0 && index < _count, "Index out of range") - if index >= InlineCapacity { - return _overflowStorage[index &- InlineCapacity] - } - return withUnsafePointer(to: _inlineStorage) { storage in - UnsafeRawPointer(storage) - .assumingMemoryBound(to: Element.self)[index] - } - } - - /// Accesses the element at `index` by copy. - /// - /// Only available for `Copyable` elements: a subscript's read accessor - /// would otherwise have to derive an address from a borrow of the inline - /// storage, which does not reliably point at the real storage. See - /// ``withElement(at:_:)`` and ``withMutableElement(at:_:)`` for the - /// noncopyable-safe access pattern this bypasses. - @inline(always) - subscript(index: Int) -> Element { - get { _copyElement(at: index) } - set { withMutableElement(at: index) { $0 = newValue } } - } - - /// Creates an array holding the elements of `sequence`, in order. - @inline(always) - init(_ sequence: S) where S.Element == Element { - self.init() - for element in sequence { - append(element) - } - } - - /// Creates an array with `count` copies of `element`. - @inline(always) - init(repeating element: Element, count: Int) { - self.init() - for _ in 0.. [Element] { - var result = [Element]() - result.reserveCapacity(_count) - let inlineCount = _inlineCount - if inlineCount > 0 { - withUnsafePointer(to: _inlineStorage) { storage in - let base = UnsafeRawPointer(storage).assumingMemoryBound(to: Element.self) - for i in 0.. -} - -/// `ProtocolEventManagerState.PendingEvent` is 328 bytes (326 rounded to stride). -@available(Network 0.1.0, *) -extension ProtocolEventManagerState.PendingEvent: NetworkInlineStorable { - typealias InlineSlot = InlineArray<41, UInt64> -} - -#endif diff --git a/Tests/QUICTests/QUICLayoutTests.swift b/Tests/QUICTests/QUICLayoutTests.swift index 4ac40462..a6741a37 100644 --- a/Tests/QUICTests/QUICLayoutTests.swift +++ b/Tests/QUICTests/QUICLayoutTests.swift @@ -31,7 +31,7 @@ final class QUICLayoutTests: XCTestCase { func testLayoutPacket() { let packetSize = 156 - let packetRecordSize = 241 + let packetRecordSize = 209 XCTAssertEqual(packetSize, MemoryLayout.size) XCTAssertEqual(packetRecordSize, MemoryLayout.size) }