Implement terminal flag (SPEC v0.6.0)

SPEC.md

@@ -52,7 +52,8 @@ The hash MUST be computed over the full message bytes: message header fields exa
 | 3 | important | Sender flags message as important. |
 | 4 | no reply | Sender will discard any reply. |
 | 5 | zlib-deflate | Message data compressed with zlib/deflate (RFC 1950/1951); _expanded size_ field present. |
-| 6–7 | reserved | Must be 0. |
+| 6 | terminal | Message is a leaf: no message may reference it via _pid_. A reply to, or add-to of, a terminal message → code 1. Enforced by hosts, unlike advisory _no reply_. |
+| 7 | reserved | MUST be 0; set → code 1. |
 
 ## 4. Common Media Types
 
@@ -101,7 +102,7 @@ Each attachment header, in order:
 
 | Field | Type | Notes |
 |-------|------|-------|
-| flags | uint8 | Bit 0 = common type (same lookup as §4). Bit 1 = zlib-deflate. Bits 2–7 reserved. |
+| flags | uint8 | Bit 0 = common type (same lookup as §4). Bit 1 = zlib-deflate. Bits 2–7 reserved, MUST be 0; set → code 1. |
 | type | uint8 + [ASCII string] | Same encoding rule as message type, using this attachment's own common type flag. |
 | filename | uint8 length + UTF-8 | < 256 bytes. Unicode letters/numbers, plus `-` `_` ` ` `.` non-consecutively, not at start/end. Unique per message (case-insensitive). |
 | size | uint32 | Byte length of this attachment's data on the wire (after compression, if zlib-deflate set). |
@@ -175,7 +176,7 @@ One message per connection. Two TCP connections used: Connection 1 (message tran
 
 ### 10.2 Sending (Host A perspective)
 
-Host A delivers iff _from_ or _add to from_ belongs to Host A's domain.
+Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. Host A MUST NOT send a message whose _pid_ references a message it holds with _terminal_ set.
 
 When _has add to_ is NOT set: perform the steps below for each unique recipient domain.
 
@@ -207,6 +208,8 @@ When _has add to_ IS set: perform the steps below for each unique participant do
    - If _has add to_ not set: ≥ 1 recipient in _to_ belongs to Host B's domain. If _has add to_ set: ≥ 1 participant (_from_, _to_, _add to from_ or _add to_) belongs to Host B's domain.
    - Common type IDs (message and attachment) are mapped.
    - _expanded size_ fields are present iff the corresponding zlib-deflate flag is set.
+   - If _has add to_: _terminal_ is NOT set (an add-to copies the original's flags, so _terminal_ means the original is terminal and cannot be referenced).
+   - No reserved flag bit is set (message bit 7, attachment bits 2–7).
 4. DNS-verify sender IP: resolve `fmsg.<sender domain>`, check Connection 1 source IP is in result set. Fail → TERMINATE.
 5. If _size_ + attachment sizes > MAX_SIZE, or total expanded size > MAX_EXPANDED_SIZE → respond code 4, close. Total expanded size uses _expanded size_ for compressed parts and _size_ for uncompressed parts.
 6. Compute DELTA = now − _time_:
@@ -218,10 +221,11 @@ When _has add to_ IS set: perform the steps below for each unique participant do
      - Verify parent stored (§11). Not found → respond code 6, close.
      - Parent time − MAX_TIME_SKEW must be before incoming time. Fail → respond code 9, close.
      - _from_ must be a participant of the parent. Fail → respond code 1, close.
+     - Parent must not be terminal. Fail → respond code 1, close.
    - **add-to set** (adding recipients):
      - pid MUST also be set. Fail → respond code 1, close.
      - Check if parent stored (§11):
-       - **Stored**: check time travel (code 9 if fail).
+       - **Stored**: parent must not be terminal (code 1 if fail); check time travel (code 9 if fail).
        - **Not stored**: if ≥ 1 recipient in _to_ or _add to_ belongs to Host B's domain, treat as full message delivery. Otherwise (Host B hosts only non-recipient participants) respond code 6 (parent not found), close.
 8. Optionally issue a CHALLENGE on Connection 2 (see §10.5).
 
@@ -289,7 +293,7 @@ An add-to message is a duplicate of the original message with these differences:
 
 An add-to message MUST be sent to every participant domain per §10.2, so all participants of the message being added to — including the original sender, when not themselves the _add to from_ — learn of the added recipients, not only the domains hosting the new recipients. This is required because a subsequent reply may reference this add-to message via _pid_, and a host can only accept a reply whose parent it holds.
 
-Add-to batches do not chain: recipients are always added to the original message; an add-to message's _pid_ MUST NOT reference another add-to message. A message therefore has 0 or more add-to batches, each a sibling branch under the original — the thread evolves as a tree.
+Add-to batches do not chain: recipients are always added to the original message; an add-to message's _pid_ MUST NOT reference another add-to message. An add-to message MUST NOT reference a terminal message (§3). A message therefore has 0 or more add-to batches, each a sibling branch under the original — the thread evolves as a tree.
 
 A recipient added by a batch and not already in _to_ is a participant of that batch message only, not of the original: their replies MUST reference the batch message via _pid_ (referencing the original would fail the participant check, §10.3 step 7) and extend the batch's branch. An address in both _to_ and _add to_ was already a participant of the original and may reply on either branch.
 

cmd/fmsgd/common_type_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// Outgoing headers encode Common Media Type IDs (SPEC §4) where the stored
+// type string has one, as FMSG-005 requires for reactions (ID 56).
+
+func commonTypeTestFields(t *testing.T) *msgFields {
+	t.Helper()
+	dir := t.TempDir()
+	bodyPath := filepath.Join(dir, "data.txt")
+	if err := os.WriteFile(bodyPath, []byte("👍"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	attPath := filepath.Join(dir, "pic.png")
+	if err := os.WriteFile(attPath, []byte("png"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return &msgFields{
+		version:  1,
+		size:     4,
+		from:     FMsgAddress{User: "alice", Domain: "example.com"},
+		to:       []FMsgAddress{{User: "bob", Domain: "example.org"}},
+		timeSent: 1754280000,
+		topic:    "types",
+		typ:      "text/plain;charset=UTF-8",
+		filepath: bodyPath,
+		attachments: []FMsgAttachmentHeader{
+			{Type: "image/png", Filename: "pic.png", Size: 3, Filepath: attPath},
+			{Type: "application/x-custom", Filename: "custom.bin", Size: 3, Filepath: attPath},
+		},
+	}
+}
+
+func TestApplyCommonTypesEncodesIDs(t *testing.T) {
+	h := commonTypeTestFields(t).originalHeader()
+	if !applyCommonTypes(h) {
+		t.Fatal("applyCommonTypes reported no change")
+	}
+	if h.Flags&FlagCommonType == 0 || h.TypeID != 56 {
+		t.Errorf("message type: flags=%#08b id=%d, want common type ID 56", h.Flags, h.TypeID)
+	}
+	if h.Attachments[0].Flags&1 == 0 || h.Attachments[0].TypeID != 38 {
+		t.Errorf("png attachment: flags=%#08b id=%d, want common type ID 38", h.Attachments[0].Flags, h.Attachments[0].TypeID)
+	}
+	if h.Attachments[1].Flags&1 != 0 {
+		t.Errorf("unmapped attachment type must stay a string, flags=%#08b", h.Attachments[1].Flags)
+	}
+	wire := h.Encode()
+	if bytes.Contains(wire, []byte("text/plain")) || bytes.Contains(wire, []byte("image/png")) {
+		t.Error("common type strings must not appear on the wire")
+	}
+	if !bytes.Contains(wire, []byte("application/x-custom")) {
+		t.Error("unmapped type string must appear on the wire")
+	}
+	if applyCommonTypes(h) {
+		t.Error("second application must be a no-op")
+	}
+}
+
+func TestEncodeForWireUsesCommonTypesForNewMessage(t *testing.T) {
+	m := commonTypeTestFields(t)
+	h, common, err := encodeForWire(m.originalHeader, deflateState{}, true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !common || h.Flags&FlagCommonType == 0 {
+		t.Errorf("new message should use common type IDs (common=%v flags=%#08b)", common, h.Flags)
+	}
+	h, common, err = encodeForWire(m.originalHeader, deflateState{}, false, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if common || h.Flags&FlagCommonType != 0 {
+		t.Errorf("commonTypes=false must keep string types (common=%v flags=%#08b)", common, h.Flags)
+	}
+}
+
+// A message whose hash was recorded before this host encoded common type IDs
+// keeps its string types, so every delivery reproduces the stored hash.
+func TestEncodeForWireKeepsRecordedForm(t *testing.T) {
+	m := commonTypeTestFields(t)
+
+	stringForm := m.originalHeader()
+	stringHash, err := stringForm.GetMessageHash()
+	if err != nil {
+		t.Fatal(err)
+	}
+	h, common, err := encodeForWire(m.originalHeader, deflateState{}, true, stringHash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if common || h.Flags&FlagCommonType != 0 {
+		t.Errorf("hash recorded in string form must keep string form (common=%v flags=%#08b)", common, h.Flags)
+	}
+	got, _ := h.GetMessageHash()
+	if !bytes.Equal(got, stringHash) {
+		t.Error("string form does not reproduce the recorded hash")
+	}
+
+	commonForm := m.originalHeader()
+	applyCommonTypes(commonForm)
+	commonHash, err := commonForm.GetMessageHash()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Equal(commonHash, stringHash) {
+		t.Fatal("forms hash identically; test no longer discriminates")
+	}
+	h, common, err = encodeForWire(m.originalHeader, deflateState{}, true, commonHash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !common || h.Flags&FlagCommonType == 0 {
+		t.Errorf("hash recorded in common form must keep common form (common=%v flags=%#08b)", common, h.Flags)
+	}
+}

cmd/fmsgd/defs.go

@@ -15,4 +15,5 @@ const (
 	FlagImportant  = fmsg.FlagImportant
 	FlagNoReply    = fmsg.FlagNoReply
 	FlagDeflate    = fmsg.FlagDeflate
+	FlagTerminal   = fmsg.FlagTerminal
 )

cmd/fmsgd/host.go

@@ -57,7 +57,7 @@ const (
 
 	RejectCodeAccept uint8 = 200
 
-	messageReservedBitsMask    uint8 = 0b11000000
+	messageReservedBitsMask    uint8 = 0b10000000 // bit 7 (SPEC §3); bit 6 is terminal
 	attachmentReservedBitsMask uint8 = 0b11111100
 )
 
@@ -349,11 +349,11 @@ func isMessageRetrievable(msg *FMsgHeader) bool {
 	if len(msg.Pid) == 0 {
 		return false
 	}
-	parentID, err := lookupMsgIdByHash(msg.Pid)
+	parentID, err := lookupMsgIdByHashFn(msg.Pid)
 	if err != nil || parentID == 0 {
 		return false
 	}
-	parentMsg, err := getMsgByID(parentID)
+	parentMsg, err := getMsgByIDFn(parentID)
 	if err != nil {
 		return false
 	}
@@ -498,6 +498,15 @@ func validateMessageFlags(c net.Conn, flags uint8) error {
 		}
 		return fmt.Errorf("reserved message flag bits set: %#08b", flags)
 	}
+	// An add-to message duplicates the original's flags, so terminal set on
+	// an add-to means the original is terminal and cannot be referenced
+	// (SPEC §10.3 step 3, §12).
+	if flags&FlagHasAddTo != 0 && flags&FlagTerminal != 0 {
+		if err := sendCode(c, RejectCodeInvalid); err != nil {
+			return err
+		}
+		return fmt.Errorf("add-to message has terminal flag set: %#08b", flags)
+	}
 	return nil
 }
 
@@ -586,7 +595,7 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) {
 	// Deliberately resolves canonical message hashes only: batches do not
 	// chain, so an add-to whose pid is another batch's hash must not resolve
 	// (SPEC §12).
-	parentID, err := lookupMsgIdByHash(h.Pid)
+	parentID, err := lookupMsgIdByHashFn(h.Pid)
 	if err != nil {
 		return h, err
 	}
@@ -595,14 +604,23 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) {
 		return handleAddToParentNotStored(c, h, hasLocalRecipient)
 	}
 
-	parentMsg, err := getMsgByID(parentID)
+	parentMsg, err := getMsgByIDFn(parentID)
 	if err != nil {
 		return h, err
 	}
 	if parentMsg == nil || !isMessageRetrievable(parentMsg) {
 		return handleAddToParentNotStored(c, h, hasLocalRecipient)
 	}
 
+	// Recipients cannot be added to a terminal message (SPEC §10.3 step 7,
+	// §12).
+	if parentMsg.Flags&FlagTerminal != 0 {
+		if err := sendCode(c, RejectCodeInvalid); err != nil {
+			return h, err
+		}
+		return h, fmt.Errorf("add-to: parent msg %d is terminal", parentID)
+	}
+
 	if parentMsg.Timestamp-FutureTimeDelta > h.Timestamp {
 		if err := sendCode(c, RejectCodeTimeTravel); err != nil {
 			return h, err
@@ -627,7 +645,7 @@ func handleAddToPath(c net.Conn, h *FMsgHeader) (*FMsgHeader, error) {
 	// A batch this host already recorded is a duplicate (SPEC §10.4 step 1).
 	// The same addresses re-issued at a new time hash differently and are a
 	// distinct batch — a new sibling branch — not a duplicate (SPEC §12).
-	recorded, err := addToBatchRecorded(parentID, batchHash)
+	recorded, err := addToBatchRecordedFn(parentID, batchHash)
 	if err != nil {
 		return h, err
 	}
@@ -652,19 +670,19 @@ func validatePidReplyPath(c net.Conn, h *FMsgHeader) error {
 		return nil
 	}
 
-	parentID, err := lookupMsgIdByHash(h.Pid)
+	parentID, err := lookupMsgIdByHashFn(h.Pid)
 	if err != nil {
 		return err
 	}
 
 	var parentMsg *FMsgHeader
 	if parentID != 0 {
-		parentMsg, err = getMsgByID(parentID)
+		parentMsg, err = getMsgByIDFn(parentID)
 	} else {
 		// A reply may reference an add-to batch message via pid (SPEC §12);
 		// its wire form is reconstructed from the stored shared message and
 		// batch fields (SPEC §11).
-		parentMsg, err = getMsgByBatchHash(h.Pid)
+		parentMsg, err = getMsgByBatchHashFn(h.Pid)
 	}
 	if err != nil {
 		return err
@@ -694,10 +712,26 @@ func validatePidReplyPath(c net.Conn, h *FMsgHeader) error {
 		}
 		return fmt.Errorf("pid reply: sender %s was not a participant of parent", h.From.ToString())
 	}
+	// A terminal message is a leaf: nothing may reference it via pid (SPEC
+	// §10.3 step 7).
+	if parentMsg.Flags&FlagTerminal != 0 {
+		if err := sendCode(c, RejectCodeInvalid); err != nil {
+			return err
+		}
+		return fmt.Errorf("pid reply: parent %s is terminal", hex.EncodeToString(h.Pid))
+	}
 
 	return nil
 }
 
+// Store lookups used by the header validation paths. Overridable in tests.
+var (
+	lookupMsgIdByHashFn  = lookupMsgIdByHash
+	getMsgByIDFn         = getMsgByID
+	getMsgByBatchHashFn  = getMsgByBatchHash
+	addToBatchRecordedFn = addToBatchRecorded
+)
+
 func readVersionOrChallenge(c net.Conn, r *bufio.Reader, h *FMsgHeader) (bool, error) {
 	v, err := r.ReadByte()
 	if err != nil {

cmd/fmsgd/host_test.go

@@ -216,6 +216,9 @@ func TestFlagConstants(t *testing.T) {
 	if FlagDeflate != 32 {
 		t.Errorf("FlagDeflate = %d, want 32 (bit 5)", FlagDeflate)
 	}
+	if FlagTerminal != 64 {
+		t.Errorf("FlagTerminal = %d, want 64 (bit 6)", FlagTerminal)
+	}
 }
 
 func encodeUInt8String(t *testing.T, s string) []byte {
@@ -482,7 +485,7 @@ func TestReadAttachmentHeadersRejectsTooBig(t *testing.T) {
 
 func TestValidateMessageFlagsRejectsReservedBits(t *testing.T) {
 	c := &testConn{}
-	err := validateMessageFlags(c, 1<<6)
+	err := validateMessageFlags(c, 1<<7)
 	if err == nil {
 		t.Fatalf("expected error for reserved message flag bit")
 	}
@@ -724,8 +727,8 @@ func TestReadAttachmentHeadersReadsExpandedSizeForCompressedAttachment(t *testin
 	})
 
 	h := &FMsgHeader{Size: 0}
-	b := []byte{1}                      // 1 attachment
-	b = append(b, 1<<1)                 // attachment flags: zlib-deflate (bit 1)
+	b := []byte{1}      // 1 attachment
+	b = append(b, 1<<1) // attachment flags: zlib-deflate (bit 1)
 	b = append(b, encodeUInt8String(t, "text/plain")...)
 	b = append(b, encodeUInt8String(t, "file.txt")...)