fix(reliability): detect in-session path-MTU black holes and step the MTU down

Source/CMakeLists.txt

@@ -52,6 +52,7 @@ set(MAFIANET_SOURCES
     src/LogCommandParser.cpp
     src/MessageFilter.cpp
     src/MmsgBatch.cpp
+    src/MtuBlackHole.cpp
     src/NatPunchthroughClient.cpp
     src/NatPunchthroughServer.cpp
     src/NatTypeDetectionClient.cpp
@@ -206,6 +207,7 @@ set(MAFIANET_HEADERS
     include/mafianet/MessageFilter.h
     include/mafianet/MessageIdentifiers.h
     include/mafianet/MmsgBatch.h
+    include/mafianet/MtuBlackHole.h
     include/mafianet/MTUSize.h
     include/mafianet/NativeFeatureIncludes.h
     include/mafianet/NativeFeatureIncludesOverrides.h

Source/include/mafianet/InternalPacket.h

@@ -122,6 +122,17 @@ struct InternalPacket : public InternalPacketFixedSizeTransmissionHeader
 	/// If the reliability type requires a receipt, then return this number with it
 	uint32_t sendReceiptSerial;
 
+	/// Sender-side bookkeeping, never transmitted: for a split fragment, the
+	/// exact bit length of the whole original message (bits, not bytes --
+	/// Send() takes bit lengths and reassembly reproduces them exactly). All
+	/// fragments of one message share their data through
+	/// refCountedData->sharedDataBlock, so together with this length the
+	/// original message can be rebuilt and re-split at a smaller size when an
+	/// in-session MTU black hole forces the negotiated MTU down (see
+	/// ReliabilityLayer::ReSplitOversizedMessages). Zero for anything that is
+	/// not a split fragment.
+	BitSize_t splitOriginalBitLength;
+
 	// Used for the resend queue
 	// Linked list implementation so I can remove from the list via a pointer, without finding it in the list
 	InternalPacket *resendPrev, *resendNext,*unreliablePrev,*unreliableNext;

Source/include/mafianet/MTUSize.h

@@ -36,16 +36,17 @@
 /// The handshake probes the path in one direction only -- the connecting peer
 /// pads ID_OPEN_CONNECTION_REQUEST_1 down the mtuSizes ladder in RakPeer.cpp and
 /// the accepting peer echoes back whatever size arrived -- and the result is
-/// then frozen for the life of the connection and applied to BOTH directions.
-/// Nothing detects a path-MTU black hole afterwards: a datagram too large for
-/// the return path is resent at the same size until the connection times out.
-///
-/// So the top rung has to be a size that survives whatever encapsulation a peer
-/// sits behind, without that peer having to discover it. 1400 clears WireGuard
+/// then applied to BOTH directions. A black hole the handshake missed (a tunnel
+/// dropping large datagrams on the return path only, or a path that shrinks
+/// mid-session) is caught later by the in-session step-down in
+/// ReliabilityLayer/MtuBlackHole.h, which walks the same ladder downward and
+/// re-splits queued messages -- but that detection costs several
+/// retransmission timeouts, so the top rung still has to be a size most
+/// tunnelled peers survive without discovering anything. 1400 clears WireGuard
 /// (1420) and typical IPSec/IKEv2 (1400) tunnels on a 1500-byte path; anything
-/// smaller is still found by the ladder. The cost is ~6% of payload per
-/// datagram on a clean path, against a class of connection failure that looks
-/// to the user like the server ignoring them.
+/// smaller is still found by the ladder or the step-down. The cost is ~6% of
+/// payload per datagram on a clean path, against a class of connection failure
+/// that looks to the user like the server ignoring them.
 ///
 /// Lowering this further is safe. RAISING it is not, on its own: two peers
 /// converge on the smaller of their two caps only because the accepting side

Source/include/mafianet/MtuBlackHole.h

@@ -0,0 +1,62 @@
+/*
+ *  Copyright (c) 2026, MafiaHub
+ *
+ *  This source code is licensed under the MIT-style license found in the
+ *  license.txt file in the root directory of this source tree.
+ */
+
+/// \file
+/// \brief In-session MTU black-hole detection.
+///
+/// The connection handshake probes the path MTU in one direction only
+/// (RakPeer.cpp pads ID_OPEN_CONNECTION_REQUEST_1 down the ladder) and the
+/// negotiated size is then applied to both directions for the life of the
+/// connection. A tunnel whose return path carries less than the probed
+/// direction -- OpenVPN and friends drop, rather than fragment, datagrams over
+/// their ceiling -- black-holes every large datagram one way while the
+/// handshake's small packets sail through. The connection establishes, then
+/// hangs on the first split payload.
+///
+/// These helpers are the portable decision logic the reliability layer uses to
+/// break that loop: recognise the black-hole signature on a resent packet and
+/// pick the next rung to drop to. Pure functions, no I/O, unit tested on every
+/// platform (Tests/Unit/MtuBlackHoleTests.cpp).
+
+#ifndef __MTU_BLACK_HOLE_H
+#define __MTU_BLACK_HOLE_H
+
+#include <stdint.h>
+
+#include "mafianet/Export.h"
+
+namespace MafiaNet {
+
+/// The MTU rung ladder, in bytes including UDP/IP headers, highest first.
+/// The same rungs the connection handshake probes (mtuSizes in RakPeer.cpp);
+/// MTU_LADDER[0] must equal MAXIMUM_MTU_SIZE so a step-down never lands on a
+/// rung the handshake could not have negotiated.
+const int MTU_LADDER_SIZE = 4;
+extern RAK_DLL_EXPORT const int MTU_LADDER[MTU_LADDER_SIZE];
+
+/// How many unacked transmissions of a too-large packet it takes before the
+/// connection's MTU steps down one rung. Resends are RTO-spaced with backoff,
+/// so this represents several round-trip times of a specific packet failing
+/// while the connection is otherwise alive -- ordinary loss does not
+/// concentrate on one packet like that.
+const uint32_t MTU_BLACKHOLE_RESEND_THRESHOLD = 4;
+
+/// The ladder rung strictly below \a currentMtuBytes, or 0 when already at or
+/// below the bottom rung.
+int RAK_DLL_EXPORT NextLowerMtu(int currentMtuBytes);
+
+/// Whether a reliable packet occupying \a requiredDatagramBytes on the wire
+/// (datagram payload plus UDP/IP headers) that has gone unacked through
+/// \a timesSent transmissions is evidence of an MTU black hole worth stepping
+/// down for. False when the packet already fits the next rung down: resending
+/// it at the same size after a step-down would change nothing on the wire, so
+/// its failures indicate loss, not a black hole.
+bool RAK_DLL_EXPORT ShouldStepDownMtu(uint32_t timesSent, int requiredDatagramBytes, int currentMtuBytes);
+
+} // namespace MafiaNet
+
+#endif // __MTU_BLACK_HOLE_H

Source/include/mafianet/ReliabilityLayer.h

@@ -177,6 +177,12 @@ class ReliabilityLayer//<ReliabilityLayer>
 	/// \param[out] the value passed to SetTimeoutTime
 	MafiaNet::TimeMS GetTimeoutTime(void);
 
+	/// The connection's current MTU in bytes, including UDP/IP headers. Starts
+	/// at the handshake-negotiated size passed to Reset() and steps down the
+	/// ladder in MtuBlackHole.h when an in-session path-MTU black hole is
+	/// detected. Never rises again for the life of the connection.
+	int GetCurrentMtuBytes(void) const;
+
 	/// Packets are read directly from the socket layer and skip the reliability layer because unconnected players do not use the reliability layer
 	/// This function takes packet data after a player has been confirmed as connected.
 	/// \param[in] buffer The socket data
@@ -324,6 +330,24 @@ class ReliabilityLayer//<ReliabilityLayer>
 	/// Split the passed packet into chunks under MTU_SIZE bytes (including headers) and save those new chunks
 	void SplitPacket( InternalPacket *internalPacket );
 
+	/// In-session path-MTU black-hole recovery: drop currentMtuBytes one rung
+	/// down the ladder in MtuBlackHole.h, shrink the congestion manager's
+	/// datagram ceiling to match, and re-split every queued message that no
+	/// longer fits. Called from Update() when a too-large reliable packet has
+	/// burnt its resend budget without an ack (ShouldStepDownMtu).
+	void StepDownMtuAfterBlackHole(void);
+
+	/// Rebuild and re-split, at the current (lowered) MTU, every queued message
+	/// with a packet too large for one datagram: split messages are
+	/// reconstructed from the shared data block their fragments reference and
+	/// re-split under a fresh splitPacketId (the receiver's partial channel for
+	/// the old id never completes and is superseded because the ordering
+	/// indices are preserved); oversized unsplit reliable messages are simply
+	/// split; oversized unsplit unreliable messages are dropped, as the network
+	/// was already free to drop them. Must only run when
+	/// packetsToSendThisUpdate is empty, since it frees queued packets.
+	void ReSplitOversizedMessages(void);
+
 	/// Insert a packet into the split packet list
 	void InsertIntoSplitPacketList( InternalPacket * internalPacket, CCTimeType time );
 
@@ -582,6 +606,10 @@ class ReliabilityLayer//<ReliabilityLayer>
 	MafiaNet::CCRakNetUDT congestionManager;
 #endif
 
+	// Current wire MTU in bytes including UDP/IP headers. Seeded from Reset()'s
+	// mtuSize, stepped down by black-hole detection. See GetCurrentMtuBytes().
+	int currentMtuBytes;
+
 
 	uint32_t unacknowledgedBytes;
 

Source/src/MtuBlackHole.cpp

@@ -0,0 +1,39 @@
+/*
+ *  Copyright (c) 2026, MafiaHub
+ *
+ *  This source code is licensed under the MIT-style license found in the
+ *  license.txt file in the root directory of this source tree.
+ */
+
+#include "mafianet/MtuBlackHole.h"
+
+#include "mafianet/MTUSize.h"
+
+namespace MafiaNet {
+
+const int MTU_LADDER[MTU_LADDER_SIZE] = {MAXIMUM_MTU_SIZE, 1280, 1024, 576};
+
+int NextLowerMtu(int currentMtuBytes)
+{
+	for (int i = 0; i < MTU_LADDER_SIZE; i++)
+	{
+		if (MTU_LADDER[i] < currentMtuBytes)
+			return MTU_LADDER[i];
+	}
+	return 0;
+}
+
+bool ShouldStepDownMtu(uint32_t timesSent, int requiredDatagramBytes, int currentMtuBytes)
+{
+	if (timesSent < MTU_BLACKHOLE_RESEND_THRESHOLD)
+		return false;
+	const int nextLower = NextLowerMtu(currentMtuBytes);
+	if (nextLower == 0)
+		return false;
+	// Only a packet that would actually shrink is evidence of a black hole:
+	// one that already fits the next rung would go out unchanged after a
+	// step-down, so its failures indicate loss, not size.
+	return requiredDatagramBytes > nextLower;
+}
+
+} // namespace MafiaNet

Source/src/RakPeer.cpp

@@ -41,6 +41,7 @@
 #include <string.h>
 #include "mafianet/GetTime.h"
 #include "mafianet/MessageIdentifiers.h"
+#include "mafianet/MtuBlackHole.h"
 #include "mafianet/DS_HuffmanEncodingTree.h"
 #include "mafianet/Rand.h"
 #include "mafianet/PluginInterface2.h"
@@ -104,10 +105,6 @@ extern void Console2GetIPAndPort(unsigned int, char *, unsigned short *, unsigne
 #endif
 
 
-static const int NUM_MTU_SIZES=4;
-
-
-
 // Probed high to low while connecting: the connecting peer pads
 // ID_OPEN_CONNECTION_REQUEST_1 to a rung and steps down when nothing comes back,
 // so the negotiated MTU is the largest rung that survived the path. Each rung is
@@ -118,7 +115,11 @@ static const int NUM_MTU_SIZES=4;
 // covers heavier or stacked encapsulation; 576 is the dial-up floor. The gap
 // from MAXIMUM_MTU_SIZE straight to 1200 that used to sit here meant a peer one
 // byte over the top rung gave up ~20% of its payload capacity to find that out.
-static const int mtuSizes[NUM_MTU_SIZES]={MAXIMUM_MTU_SIZE, 1280, 1024, 576};
+//
+// The ladder itself lives in MtuBlackHole.h: the in-session black-hole
+// step-down (ReliabilityLayer) walks the same rungs this handshake probes.
+static const int NUM_MTU_SIZES=MafiaNet::MTU_LADDER_SIZE;
+static const int *const mtuSizes=MafiaNet::MTU_LADDER;
 
 // How many connection attempts are spent on each rung of mtuSizes before
 // stepping down.
@@ -6279,6 +6280,17 @@ bool RakPeer::RunUpdateCycle(BitStream &updateBitStream )
 			else
 				remoteSystem->reliabilityLayer.Update( remoteSystem->rakNetSocket, systemAddress, remoteSystem->MTUSize, timeNS, maxOutgoingBPS, pluginListNTS, &rnr, updateBitStream ); // systemAddress only used for the internet simulator test
 
+			// The reliability layer steps the negotiated MTU down when it
+			// detects an in-session path-MTU black hole; keep the value
+			// GetMTUSize() reports in sync with what is actually on the wire.
+			// Written only on an actual step-down: GetMTUSize() reads this
+			// field from the user thread without synchronization (as it always
+			// has for the connect-time write), so avoid turning it into a
+			// continuously-written field.
+			const int reliabilityMtu = remoteSystem->reliabilityLayer.GetCurrentMtuBytes();
+			if (remoteSystem->MTUSize != reliabilityMtu)
+				remoteSystem->MTUSize = reliabilityMtu;
+
 			// Check for failure conditions
 			if ( remoteSystem->reliabilityLayer.IsDeadConnection() ||
 				((remoteSystem->connectMode==RemoteSystemStruct::DISCONNECT_ASAP || remoteSystem->connectMode==RemoteSystemStruct::DISCONNECT_ASAP_SILENTLY) && remoteSystem->reliabilityLayer.IsOutgoingDataWaiting()==false) ||