-
Notifications
You must be signed in to change notification settings - Fork 581
[typemap] Stream typemap fingerprints and PE output #12602
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
simonrozsival
merged 19 commits into
main
from
simonrozsival-typemap-optimization-retry
Sep 10, 2026
Merged
Changes from all commits
Commits
Show all changes
19 commits
Select commit
Hold shift + click to select a range
a6de122
Optimize trimmable typemap generation
simonrozsival fa94514
Skip unchanged typemap assembly emission
simonrozsival 51e0fdf
Skip unchanged typemap assembly emission
simonrozsival 59dcd2e
Merge commit '51e0fdf677d17da24290d3e8043897f64069d0aa' into simonroz…
simonrozsival 633d8c7
Preserve typemap fingerprints incrementally
simonrozsival 3ebb677
Reduce typemap signature encoder allocations
simonrozsival e8ac902
Reuse metadata signature encoder
simonrozsival e055229
Compute typemap fingerprints in a single streaming pass
simonrozsival d5ee47f
Return generated typemap assemblies without a second buffer
simonrozsival 3eb5249
Address code review feedback
simonrozsival 37b87f9
Fix typemap incremental test source update
simonrozsival efaed88
Update test logger for main
simonrozsival e483c30
Merge main into typemap optimization
simonrozsival 26fc624
Improve typemap streaming implementation
simonrozsival 8fc72bb
Merge main into typemap optimization
simonrozsival eaf5b1f
Merge main into typemap optimization
simonrozsival 949249b
Merge main into typemap optimization
simonrozsival e9e0b28
Relocate typemap JNI rewrite tests
simonrozsival b794125
Merge main into typemap optimization
simonrozsival File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
112 changes: 112 additions & 0 deletions
112
....Build.Tasks/Tests/Microsoft.Android.Build.Tasks.Tests/JniAssemblyRewriterTypeMapTests.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| extern alias xamarinbuildtasks; | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Collections.Immutable; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Reflection.Metadata; | ||
| using System.Reflection.PortableExecutable; | ||
| using Microsoft.Android.Sdk.TrimmableTypeMap; | ||
| using Microsoft.Build.Framework; | ||
| using Microsoft.Build.Utilities; | ||
| using NUnit.Framework; | ||
| using FieldRvaTable = xamarinbuildtasks::Xamarin.Android.Tasks.JniRemapping.FieldRvaTable; | ||
| using JniAssemblyRewriter = xamarinbuildtasks::Xamarin.Android.Tasks.JniRemapping.JniAssemblyRewriter; | ||
| using JniRewriteResult = xamarinbuildtasks::Xamarin.Android.Tasks.JniRemapping.JniRewriteResult; | ||
| using R8Mapping = xamarinbuildtasks::Xamarin.Android.Tasks.JniRemapping.R8Mapping; | ||
|
|
||
| namespace Xamarin.Android.Build.Tests; | ||
|
|
||
| [TestFixture] | ||
| [Parallelizable (ParallelScope.Children)] | ||
| public class JniAssemblyRewriterTypeMapTests | ||
| { | ||
| [Test] | ||
| public void RewritesGeneratedTypeMapWithOwnerSpecificMethodNames () | ||
| { | ||
| byte [] source = GenerateTypeMapWithSharedMethodName (); | ||
| var warnings = new List<BuildWarningEventArgs> (); | ||
|
|
||
| JniRewriteResult result = Rewrite (source, Mapping ( | ||
| "test.First -> a.b.First:\n" + | ||
| " void n_Run() -> a\n" + | ||
| "test.Second -> a.b.Second:\n" + | ||
| " void n_Run() -> b\n"), warnings); | ||
|
|
||
| CollectionAssert.AreEquivalent (new [] { "a", "b", "()V" }, ReadUtf8Values (result.Image)); | ||
| CollectionAssert.DoesNotContain (warnings.Select (warning => warning.Code).ToArray (), "XA4326"); | ||
| } | ||
|
|
||
| [Test] | ||
| public void RewritesGeneratedTypeMapWithMappedAndUnmappedMethodNames () | ||
| { | ||
| byte [] source = GenerateTypeMapWithSharedMethodName (); | ||
| var warnings = new List<BuildWarningEventArgs> (); | ||
|
|
||
| JniRewriteResult result = Rewrite (source, Mapping ( | ||
| "test.First -> a.b.First:\n" + | ||
| " void n_Run() -> a\n" + | ||
| "test.Second -> test.Second:\n"), warnings); | ||
|
|
||
| CollectionAssert.AreEquivalent (new [] { "a", "n_Run", "()V" }, ReadUtf8Values (result.Image)); | ||
| CollectionAssert.DoesNotContain (warnings.Select (warning => warning.Code).ToArray (), "XA4326"); | ||
| } | ||
|
|
||
| static JniRewriteResult Rewrite (byte [] sourceImage, R8Mapping mapping, IList<BuildWarningEventArgs> warnings) | ||
| { | ||
| var engine = new MockBuildEngine (TestContext.Out, warnings: warnings); | ||
| var log = new TaskLoggingHelper (engine, nameof (JniAssemblyRewriterTypeMapTests)); | ||
| return JniAssemblyRewriter.Rewrite (sourceImage, mapping, log); | ||
| } | ||
|
|
||
| static R8Mapping Mapping (string text) => R8Mapping.Parse (new StringReader (text)); | ||
|
|
||
| static byte [] GenerateTypeMapWithSharedMethodName () | ||
| { | ||
| var peers = new [] { | ||
| CreatePeer ("test/First", "Test.First"), | ||
| CreatePeer ("test/Second", "Test.Second"), | ||
| }; | ||
| using var stream = new MemoryStream (); | ||
| new TypeMapAssemblyGenerator (new Version (11, 0, 0, 0)).Generate (peers, stream, "OwnerSpecificNames"); | ||
| return stream.ToArray (); | ||
| } | ||
|
|
||
| static JavaPeerInfo CreatePeer (string javaName, string managedName) | ||
| { | ||
| int separator = managedName.LastIndexOf ('.'); | ||
| return new JavaPeerInfo { | ||
| JavaName = javaName, | ||
| CompatJniName = javaName, | ||
| ManagedTypeName = managedName, | ||
| ManagedTypeNamespace = managedName.Substring (0, separator), | ||
| ManagedTypeShortName = managedName.Substring (separator + 1), | ||
| AssemblyName = "TestAsm", | ||
| DoNotGenerateAcw = false, | ||
| ActivationCtor = new ActivationCtorInfo { | ||
| DeclaringTypeName = managedName, | ||
| DeclaringAssemblyName = "TestAsm", | ||
| Style = ActivationCtorStyle.XamarinAndroid, | ||
| }, | ||
| MarshalMethods = [ | ||
| new MarshalMethodInfo { | ||
| JniName = "run", | ||
| NativeCallbackName = "n_Run", | ||
| JniSignature = "()V", | ||
| ManagedMethodName = "Run", | ||
| }, | ||
| ], | ||
| }; | ||
| } | ||
|
|
||
| static string [] ReadUtf8Values (byte [] image) | ||
| { | ||
| using var peReader = new PEReader (ImmutableArray.Create (image)); | ||
| MetadataReader reader = peReader.GetMetadataReader (); | ||
| return FieldRvaTable.Read (peReader, reader).Entries | ||
| .Select (entry => entry.Utf8Value) | ||
| .OfType<string> () | ||
| .ToArray (); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
147 changes: 147 additions & 0 deletions
147
src/Microsoft.Android.Sdk.TrimmableTypeMap/Generator/BlobBuilderStream.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Reflection.Metadata; | ||
|
|
||
| namespace Microsoft.Android.Sdk.TrimmableTypeMap; | ||
|
|
||
| /// <summary> | ||
| /// A read-only, seekable <see cref="Stream"/> over the chunks a <see cref="BlobBuilder"/> already | ||
| /// holds, so a serialised PE image can be hashed and copied to disk without being duplicated into | ||
| /// a second contiguous buffer. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// Only the chunk arrays are retained — the <see cref="BlobBuilder"/> itself and the metadata | ||
| /// graph that produced it stay collectible — so the live byte count matches what a | ||
| /// <see cref="MemoryStream"/> copy would have held, without the transient second copy. | ||
| /// The source builder must not be cleared, freed, or reused after this stream is created because | ||
| /// the stream retains references to its chunk arrays. | ||
| /// </remarks> | ||
| sealed class BlobBuilderStream : Stream | ||
| { | ||
| readonly ArraySegment<byte> [] segments; | ||
| readonly long [] segmentStarts; | ||
| readonly long length; | ||
| long position; | ||
| int cursor; | ||
|
|
||
| public BlobBuilderStream (BlobBuilder builder) | ||
| { | ||
| _ = builder ?? throw new ArgumentNullException (nameof (builder)); | ||
|
|
||
| var collected = new List<ArraySegment<byte>> (); | ||
| foreach (var blob in builder.GetBlobs ()) { | ||
| var bytes = blob.GetBytes (); | ||
| if (bytes.Count == 0 || bytes.Array is null) { | ||
| continue; | ||
| } | ||
| collected.Add (bytes); | ||
| } | ||
|
|
||
| segments = collected.ToArray (); | ||
| segmentStarts = new long [segments.Length + 1]; | ||
| long total = 0; | ||
| for (int i = 0; i < segments.Length; i++) { | ||
| segmentStarts [i] = total; | ||
| total += segments [i].Count; | ||
| } | ||
| segmentStarts [segments.Length] = total; | ||
| length = total; | ||
| } | ||
|
|
||
| public override bool CanRead => true; | ||
|
|
||
| public override bool CanSeek => true; | ||
|
|
||
| public override bool CanWrite => false; | ||
|
|
||
| public override long Length => length; | ||
|
|
||
| public override long Position { | ||
| get => position; | ||
| set { | ||
| if (value < 0) { | ||
| throw new ArgumentOutOfRangeException (nameof (value)); | ||
| } | ||
| position = value; | ||
| } | ||
| } | ||
|
|
||
| public override void Flush () | ||
| { | ||
| } | ||
|
|
||
| public override int Read (byte [] buffer, int offset, int count) | ||
| { | ||
| if (buffer is null) { | ||
| throw new ArgumentNullException (nameof (buffer)); | ||
| } | ||
| if (offset < 0) { | ||
| throw new ArgumentOutOfRangeException (nameof (offset)); | ||
| } | ||
| if (count < 0) { | ||
| throw new ArgumentOutOfRangeException (nameof (count)); | ||
| } | ||
| if (buffer.Length - offset < count) { | ||
| throw new ArgumentException ("The buffer is too small for the requested range."); | ||
| } | ||
|
|
||
| int copied = 0; | ||
| while (count > 0 && position < length) { | ||
| int index = FindSegment (position); | ||
| var segment = segments [index]; | ||
| int within = (int) (position - segmentStarts [index]); | ||
| int available = segment.Count - within; | ||
| int toCopy = Math.Min (available, count); | ||
| var source = segment.Array ?? throw new InvalidOperationException ("Blob segment has no backing array."); | ||
| Buffer.BlockCopy (source, segment.Offset + within, buffer, offset, toCopy); | ||
| position += toCopy; | ||
| offset += toCopy; | ||
| count -= toCopy; | ||
| copied += toCopy; | ||
| } | ||
| return copied; | ||
| } | ||
|
|
||
| public override long Seek (long offset, SeekOrigin origin) | ||
| { | ||
| long target = origin switch { | ||
| SeekOrigin.Begin => offset, | ||
| SeekOrigin.Current => position + offset, | ||
| SeekOrigin.End => length + offset, | ||
| _ => throw new ArgumentOutOfRangeException (nameof (origin)), | ||
| }; | ||
| if (target < 0) { | ||
| throw new IOException ("Cannot seek before the beginning of the stream."); | ||
| } | ||
| position = target; | ||
| return position; | ||
| } | ||
|
|
||
| public override void SetLength (long value) => throw new NotSupportedException (); | ||
|
|
||
| public override void Write (byte [] buffer, int offset, int count) => throw new NotSupportedException (); | ||
|
|
||
| int FindSegment (long offset) | ||
| { | ||
| // Reads are overwhelmingly sequential, so try the last used chunk first. | ||
| if (cursor < segments.Length && offset >= segmentStarts [cursor] && offset < segmentStarts [cursor + 1]) { | ||
| return cursor; | ||
| } | ||
|
|
||
| int low = 0; | ||
| int high = segments.Length - 1; | ||
| while (low <= high) { | ||
| int middle = low + ((high - low) / 2); | ||
| if (offset < segmentStarts [middle]) { | ||
| high = middle - 1; | ||
| } else if (offset >= segmentStarts [middle + 1]) { | ||
| low = middle + 1; | ||
| } else { | ||
| cursor = middle; | ||
| return middle; | ||
| } | ||
| } | ||
| throw new ArgumentOutOfRangeException (nameof (offset)); | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.