[#130] Support SerializedFile v26 (Unity 6.7)

Analyzer.Tests/FileDetectionTests.cs

@@ -188,15 +188,19 @@ public void TryParseMetadata_VersionTooOld_ReturnsFalseWithMessage()
     [Test]
     public void TryParseMetadata_VersionTooNew_ReturnsFalseWithMessage()
     {
-        var headerInfo = new SerializedFileInfo { Version = 24 };
+        // Far enough ahead that this does not need revisiting every time a Unity release adds a
+        // version. TryParseMetadata is given the header directly, so the detector's own plausibility
+        // range does not apply here.
+        var headerInfo = new SerializedFileInfo { Version = 99 };
 
         bool result = SerializedFileDetector.TryParseMetadata("irrelevant", headerInfo, out var metadata, out var errorMessage);
 
         Assert.IsFalse(result);
         Assert.IsNull(metadata);
         Assert.IsNotNull(errorMessage);
-        Assert.That(errorMessage, Does.Contain("24"), "Error should mention the actual version");
-        Assert.That(errorMessage, Does.Contain("23"), "Error should mention the maximum supported version");
+        Assert.That(errorMessage, Does.Contain("99"), "Error should mention the actual version");
+        Assert.That(errorMessage, Does.Contain(SerializedFileDetector.MaxMetadataParseVersion.ToString()),
+            "Error should mention the maximum supported version");
         Assert.That(errorMessage, Does.Contain("UnityDataTool"), "Error should mention UnityDataTool");
     }
 
@@ -297,6 +301,63 @@ public void TryParseMetadata_PlayerNoTypeTreeLevel1_ReturnsExpectedValues()
         }
     }
 
+    [Test]
+    public void TryParseMetadata_V26PlayerFile_ReturnsSharedSubtreeTable()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "PlayerWithTypeTreesV26", "sharedassets1.assets");
+
+        Assert.IsTrue(SerializedFileDetector.TryDetectSerializedFile(testFile, out var headerInfo));
+        Assert.That(headerInfo.Version, Is.EqualTo(26u));
+
+        bool result = SerializedFileDetector.TryParseMetadata(testFile, headerInfo, out var metadata, out var errorMessage);
+        Assert.IsTrue(result, $"Metadata parsing should succeed. Error: {errorMessage}");
+
+        Assert.That(metadata.UnityVersion, Is.EqualTo("6000.7.0b2"));
+        Assert.That(metadata.TypeTreeCount, Is.EqualTo(2), "PreloadData and MonoBehaviour");
+        Assert.That(metadata.SerializedReferenceTypeTreeCount, Is.EqualTo(1));
+
+        // The shared subtree table follows m_RefTypes and is what version 26 adds to the metadata.
+        Assert.That(metadata.SharedSubtreeCount, Is.EqualTo(7));
+        Assert.That(metadata.SharedSubtrees.Length, Is.EqualTo(metadata.SharedSubtreeCount));
+
+        foreach (var subtree in metadata.SharedSubtrees)
+        {
+            Assert.IsTrue(subtree.Inline, "This build embeds its subtree blobs rather than extracting them");
+            Assert.That(subtree.SerializedSize, Is.GreaterThan(0));
+            Assert.IsFalse(subtree.ContentHash.IsZero, "A subtree is keyed by the hash of its content");
+        }
+    }
+
+    // The stamp at the head of a TypeTree blob tells the two version spaces apart: up to and
+    // including 23 it repeats the SerializedFile version, and from 24 TypeTrees are versioned
+    // independently starting at 32.
+    [TestCase("PlayerWithTypeTrees/sharedassets1.assets", 0u, TestName = "TypeTreeFormatVersion_V22_HasNoStamp")]
+    [TestCase("AssetBundleTypeTreeVariations/v23_Inline/prefab_with_serializedreference.serializedfile", 23u, TestName = "TypeTreeFormatVersion_V23_StampsTheFileVersion")]
+    [TestCase("PlayerWithTypeTreesV26/sharedassets1.assets", 33u, TestName = "TypeTreeFormatVersion_V26_StampsTheIndependentVersion")]
+    public void TryParseMetadata_ReportsTypeTreeFormatVersion(string relativePath, uint expected)
+    {
+        var testFile = Path.Combine(m_TestDataPath, relativePath.Replace('/', Path.DirectorySeparatorChar));
+
+        Assert.IsTrue(SerializedFileDetector.TryDetectSerializedFile(testFile, out var headerInfo));
+        Assert.IsTrue(SerializedFileDetector.TryParseMetadata(testFile, headerInfo, out var metadata, out _));
+
+        foreach (var entry in metadata.TypeTrees)
+            Assert.That(entry.TypeTreeFormatVersion, Is.EqualTo(expected), $"persistentTypeID={entry.PersistentTypeID}");
+    }
+
+    [Test]
+    public void TryParseMetadata_BeforeVersion26_HasNoSharedSubtreeTable()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "AssetBundleTypeTreeVariations", "v23_Inline",
+            "prefab_with_serializedreference.serializedfile");
+
+        Assert.IsTrue(SerializedFileDetector.TryDetectSerializedFile(testFile, out var headerInfo));
+        Assert.IsTrue(SerializedFileDetector.TryParseMetadata(testFile, headerInfo, out var metadata, out _));
+
+        Assert.That(metadata.SharedSubtreeCount, Is.EqualTo(0));
+        Assert.That(metadata.SharedSubtrees, Is.Empty);
+    }
+
     [Test]
     public void TryParseMetadata_V22PrefabWithSerializedReference_ReturnsExpectedTypeTreeData()
     {

Analyzer/AnalyzerTool.cs

@@ -120,12 +120,13 @@ public int Analyze(AnalyzeOptions options)
                         Console.Error.WriteLine($"Skipped (no TypeTrees): {relativePath}");
                         countNoTypeTrees++;
                     }
-                    catch (SerializedFileOpenException)
+                    catch (SerializedFileOpenException e)
                     {
                         // Expected failure — the file content could not be parsed.
                         // Don't print a stack trace; it adds no value for this known failure mode.
                         EraseProgressLine();
                         Console.Error.WriteLine($"Failed to open: {relativePath}");
+                        Console.Error.WriteLine($"  {e.Message}");
                         countFailures++;
                     }
                     catch (AnalyzeDuplicateException e)

Analyzer/PPtrAndCrcProcessor.cs

@@ -51,6 +51,7 @@ public class PPtrAndCrcProcessor : IDisposable
 
     // State for the object currently being processed, (re)initialized by each Process() call.
     private long m_Offset;       // current read position within m_Reader
+    private long m_ObjectEnd;    // end of the object being processed; bounds the registry frame read
     private long m_ObjectId;     // analyzer id of the object being processed, passed to the callback
     private uint m_Crc32;        // CRC accumulated so far for this object
 
@@ -85,17 +86,29 @@ public void Dispose()
         m_resourceReaders.Clear();
     }
 
-    // Walks the serialized object rooted at `node`, whose data starts at `offset` in the reader,
-    // emitting every PPtr through the callback. Returns a CRC32 fingerprint of the object's content
-    // (0 when CRC is disabled). `objectId` is the analyzer id of this object, forwarded to the callback.
-    public uint Process(long objectId, long offset, TypeTreeNode node)
+    // Walks the serialized object rooted at `node`, whose data starts at `offset` in the reader and
+    // is `size` bytes long, emitting every PPtr through the callback. Returns a CRC32 fingerprint of
+    // the object's content (0 when CRC is disabled). `objectId` is the analyzer id of this object,
+    // forwarded to the callback.
+    public uint Process(long objectId, long offset, long size, TypeTreeNode node)
     {
         m_Offset = offset;
+        m_ObjectEnd = offset + size;
         m_ObjectId = objectId;
         m_Crc32 = 0;
 
         foreach (var child in node.Children)
         {
+            // A version 3 registry sits here rather than in a node (see
+            // ProcessManagedReferenceRegistry). It is not part of the field that follows it, so its
+            // references get their own path root, named as the node-described versions are.
+            if (child.HasSerializedRefs)
+            {
+                m_StringBuilder.Clear();
+                m_StringBuilder.Append("references.");
+                ProcessManagedReferenceFrame();
+            }
+
             m_StringBuilder.Clear();
             m_StringBuilder.Append(child.Name);
             ProcessNode(child, false);
@@ -104,6 +117,29 @@ public uint Process(long objectId, long offset, TypeTreeNode node)
         return m_Crc32;
     }
 
+    // Walks a version 3 registry: the frame's header and tables are CRC'd as the raw bytes they
+    // are, then each entry's data is walked through its own type tree, exactly as the node-described
+    // versions are.
+    private void ProcessManagedReferenceFrame()
+    {
+        var registry = ManagedReferenceRegistry.ReadFrame(m_Reader, m_Offset, m_ObjectEnd - m_Offset);
+        var frameStart = m_Offset;
+
+        // Everything ahead of the first blob is the header and the two tables.
+        AppendCrc(frameStart, (int)(registry.BlobsOffset - frameStart));
+
+        foreach (var entry in registry.Entries)
+        {
+            if (entry.IsNull)
+                continue;
+
+            m_Offset = entry.DataOffset;
+            ProcessRefTypeData(entry.Rid, entry.ClassName, entry.Namespace, entry.AssemblyName);
+        }
+
+        m_Offset = frameStart + registry.FrameSize;
+    }
+
     private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry)
     {
         if (node.IsBasicType)
@@ -268,10 +304,10 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry, bo
     }
 
     // A ManagedReferenceRegistry holds the [SerializeReference] instances owned by this object.
-    // In YAML/JSON it is the "references:" section that always appears at the end of a
-    // MonoBehaviour/ScriptableObject. Each instance is stored here exactly once; the fields that
-    // point at it (elsewhere in the object) only store its "rid", so shared instances and cycles
-    // collapse to the same rid.
+    // In YAML/JSON it is the "references:" section of a MonoBehaviour/ScriptableObject, which
+    // appears at the end of the object up to Unity 6.6 and ahead of the referencing fields from
+    // 6.7. Each instance is stored here exactly once; the fields that point at it (elsewhere in the
+    // object) only store its "rid", so shared instances and cycles collapse to the same rid.
     //
     // Given this C# source:
     //
@@ -304,10 +340,21 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry, bo
     // a different TypeTree for every entry (see ProcessManagedReferenceData) - which is exactly why
     // finding references inside the registry is so much more involved than for the rest of the object.
     //
-    // Two on-disk versions exist:
+    // Three on-disk versions exist:
     //   version 1 - entries stored back to back and terminated by a sentinel type (see
     //               ProcessManagedReferenceData); the rid is implied by position.
     //   version 2 - entries stored as a "RefIds" array, each element carrying its own rid.
+    //   version 3 - from SerializedFile version 25 (Unity 6.7). No node describes it at all: the
+    //               registry is a self-delimiting frame of raw bytes leading the C# class's own
+    //               data, so it sits after the built-in fields (m_GameObject, m_Name, ...) and
+    //               before the first field the script declares - which is the field flagged
+    //               HasSerializedRefs, whether or not that field is itself a reference. It holds a
+    //               table of type names and a table of records indexing into it; a record can be a
+    //               null entry, which v1 and v2 could not express. Only a root object's data
+    //               carries a frame: the same TypeTree used to lay out an instance's data inside
+    //               the registry keeps the flag but has no frame, which is why every walker honours
+    //               it only while iterating a root object's fields.
+    //               ManagedReferenceRegistry reads it; see ProcessManagedReferenceFrame here.
     private void ProcessManagedReferenceRegistry(TypeTreeNode node)
     {
         if (node.Children.Count < 2)
@@ -393,18 +440,24 @@ bool ProcessManagedReferenceData(TypeTreeNode refTypeNode, long rid)
             return false;
         }
 
-        // The data block follows the referenced type's own TypeTree, not this object's, so look it
-        // up by FQN and walk it (isInManagedReferenceRegistry = true so we don't re-enter the registry).
+        ProcessRefTypeData(rid, className, namespaceName, assemblyName);
+
+        return true;
+    }
+
+    // The data block follows the referenced type's own TypeTree, not the containing object's, so it
+    // is looked up by FQN and walked with isInManagedReferenceRegistry set, which keeps the walk
+    // from re-entering the registry.
+    private void ProcessRefTypeData(long rid, string className, string namespaceName, string assemblyName)
+    {
         var refTypeTypeTree = m_SerializedFile.GetRefTypeTypeTreeRoot(className, namespaceName, assemblyName);
 
-        var size = m_StringBuilder.Length;
+        var pathLength = m_StringBuilder.Length;
         m_StringBuilder.Append("rid(");
         m_StringBuilder.Append(rid);
         m_StringBuilder.Append(").data");
         ProcessNode(refTypeTypeTree, true);
-        m_StringBuilder.Remove(size, m_StringBuilder.Length - size);
-
-        return true;
+        m_StringBuilder.Remove(pathLength, m_StringBuilder.Length - pathLength);
     }
 
     private void ExtractPPtr(string referencedType)

Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs

@@ -180,6 +180,11 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
         // VFS path here may be a real file or an entry inside a mounted archive.
         using (var detectStream = new UnityFileStream(fullPath))
         {
+            // A version this build cannot read reaches the native loader as a generic failure, so
+            // say which version it is here rather than leaving the user with "may be corrupted".
+            if (SerializedFileDetector.IsVersionUnsupported(detectStream, out var versionError))
+                throw new SerializedFileOpenException(fullPath, versionError);
+
             if (SerializedFileDetector.IsMissingTypeTrees(detectStream))
                 throw new SerializedFileOpenException(fullPath, missingTypeTrees: true);
         }
@@ -319,7 +324,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
                     m_TypeSet.Add(obj.TypeId);
                 }
 
-                var randomAccessReader = new RandomAccessReader(sf, root, reader, offset);
+                var randomAccessReader = new RandomAccessReader(sf, root, reader, offset, objectSize: obj.Size);
 
                 string name = string.Empty;
                 long streamDataSize = 0;
@@ -356,7 +361,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
                 // still resolves referenced object ids (AddReference skips the insert).
                 if (!m_SkipReferences || !m_SkipCrc)
                 {
-                    crc32 = pptrReader.Process(currentObjectId, offset, root);
+                    crc32 = pptrReader.Process(currentObjectId, offset, obj.Size, root);
                 }
 
                 // convert this to the new syntax

Documentation/command-dump.md

@@ -183,6 +183,81 @@ ID: -8138362113332287275 (ClassID: 135) SphereCollider
     z float 0
 ```
 
+### `[SerializeReference]` fields
+
+A field marked `[SerializeReference]` in C# appears in the dump with the type **`managedReference`**,
+and its value is not the object - it is a number called a **`rid`** (reference id). The objects
+themselves are listed together in a `references` section, once each, with their concrete C# type and
+their field values.
+
+Take this script:
+
+```csharp
+public class Inventory : MonoBehaviour
+{
+    [Serializable]
+    public class Item
+    {
+        public string name;
+        public int count;
+    }
+
+    [SerializeReference] public Item primary;
+    [SerializeReference] public Item backup;
+    [SerializeReference] public Item spare;
+}
+```
+
+with `primary` and `backup` both assigned the *same* `Item`, and `spare` left null. Dumping it:
+
+```
+ID: 3862108129085620391 (ClassID: 114) MonoBehaviour
+  m_GameObject (PPtr<GameObject>)
+    m_FileID (int) 0
+    m_PathID (SInt64) -5904263129458716409
+  m_Enabled (UInt8) 1
+  m_Script (PPtr<MonoScript>)
+    m_FileID (int) 1
+    m_PathID (SInt64) 1197423208934291241
+  m_Name (string)
+  references (ManagedReferenceRegistry)
+    version (int) 3
+    rid(-2) ReferencedObject
+      null
+    rid(3218405420927320064) ReferencedObject
+      type (ReferencedManagedType)
+        class (string) Inventory/Item
+        ns (string)
+        asm (string) Assembly-CSharp
+      data ReferencedObjectData
+        name (string) Health potion
+        count (int) 3
+  primary (managedReference)
+    rid (SInt64) 3218405420927320064
+  backup (managedReference)
+    rid (SInt64) 3218405420927320064
+  spare (managedReference)
+    rid (SInt64) -2
+```
+
+Reading it:
+
+* **`primary` and `backup` show the same `rid`.** They are two references to one object, so the
+  object is listed once and the sharing is visible - this is the whole point of
+  `[SerializeReference]` over a plain serialized field, which would have stored two independent
+  copies.
+* **`spare` shows `rid (SInt64) -2`**, the marker for null, and the matching entry is listed as
+  `null` with no type or data.
+* **`class` is the concrete runtime type**, which can be a subclass of the field's declared type -
+  that is what `[SerializeReference]` is for. Nested classes use a `/`, as in `Inventory/Item`.
+* The entries are listed in the order the file stores them, which is not necessarily the order the
+  fields appear in.
+
+The `version` line describes how the registry is stored rather than anything about your data. It is
+`3` for content built with Unity 6.7 or newer and `2` before that, and in older files the
+`references` section appears *after* the fields instead of before them. The entries mean the same
+thing either way.
+
 **Refer to the [TextDumper documentation](textdumper.md) for detailed output format explanation.**
 
 ---