diff --git a/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs b/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs index aebd610a2..c2973306d 100644 --- a/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs +++ b/src/ImageBuilder.Tests/Helpers/InMemoryFileSystem.cs @@ -18,6 +18,7 @@ namespace Microsoft.DotNet.ImageBuilder.Tests.Helpers; internal sealed class InMemoryFileSystem : IFileSystem { private readonly Dictionary _files = []; + private readonly Dictionary _unixFileModes = []; private readonly HashSet _directories = []; /// @@ -83,6 +84,9 @@ public Task WriteAllTextAsync(string path, string? contents, CancellationToken c public Stream CreateFile(string path) => new CommitOnDisposeStream(this, path); + public void SetUnixFileMode(string path, UnixFileMode mode) => + _unixFileModes[path] = mode; + public byte[] ReadAllBytes(string path) { FilesRead.Add(path); @@ -189,6 +193,12 @@ public string GetFileText(string path) => /// public byte[] GetFileBytes(string path) => _files[path]; + /// + /// Gets the Unix mode assigned to a file, for test assertions. + /// + public UnixFileMode? GetUnixFileMode(string path) => + _unixFileModes.TryGetValue(path, out UnixFileMode mode) ? mode : null; + /// /// Writable stream returned by that commits its contents to the /// in-memory store when disposed, mirroring the returned by diff --git a/src/ImageBuilder.Tests/UpdateCommandTests.cs b/src/ImageBuilder.Tests/UpdateCommandTests.cs index 0b23de4ff..c6a7081e9 100644 --- a/src/ImageBuilder.Tests/UpdateCommandTests.cs +++ b/src/ImageBuilder.Tests/UpdateCommandTests.cs @@ -20,6 +20,23 @@ namespace Microsoft.DotNet.ImageBuilder.Tests; [TestClass] public class UpdateCommandTests { + // Unix file mode 0755: rwxr-xr-x. + private const UnixFileMode ExecutableFileMode = + UnixFileMode.UserRead | + UnixFileMode.UserWrite | + UnixFileMode.UserExecute | + UnixFileMode.GroupRead | + UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | + UnixFileMode.OtherExecute; + + // Unix file mode 0644: rw-r--r--. + private const UnixFileMode RegularFileMode = + UnixFileMode.UserRead | + UnixFileMode.UserWrite | + UnixFileMode.GroupRead | + UnixFileMode.OtherRead; + // Use the platform's directory separator for the fake root so that paths derived via // Path.Combine and Path.GetDirectoryName stay consistent (Path.GetDirectoryName normalizes // a leading '/' to '\' on Windows, which would otherwise not match the in-memory entries). @@ -58,6 +75,34 @@ public async Task UpdateCommand_WritesAllEmbeddedFiles() renderedDestinations.Count.ShouldBe(1); } + [TestMethod] + public async Task UpdateCommand_PreservesUnixFileModes() + { + string[] expectedExecutableRelativePaths = + [ + "skill-helpers/Get-BuildLog.ps1", + "skill-helpers/Get-FailingPipelines.ps1", + "skill-helpers/Show-BuildTimeline.ps1", + "skill-helpers/Show-PullRequestComments.ps1" + ]; + InMemoryFileSystem fileSystem = CreateRepoFileSystem(); + fileSystem.AddDirectory(s_outputPath); + UpdateCommand command = CreateCommand(fileSystem); + + await command.ExecuteAsync(); + + foreach (string relativePath in InfrastructureContent.GetRelativePaths()) + { + string destination = PathHelper.SafeCombine(s_outputPath, relativePath); + UnixFileMode expectedMode = expectedExecutableRelativePaths.Contains(relativePath) + ? ExecutableFileMode + : RegularFileMode; + InfrastructureContent.GetUnixFileMode(relativePath).ShouldBe(expectedMode); + fileSystem.GetUnixFileMode(destination).ShouldBe( + OperatingSystem.IsWindows() ? null : expectedMode); + } + } + [TestMethod] public async Task UpdateCommand_ImageBuilderRefProvided_RendersDockerImagesTemplate() { diff --git a/src/ImageBuilder/Commands/UpdateCommand.cs b/src/ImageBuilder/Commands/UpdateCommand.cs index 3a1420181..a625fc4c8 100644 --- a/src/ImageBuilder/Commands/UpdateCommand.cs +++ b/src/ImageBuilder/Commands/UpdateCommand.cs @@ -130,6 +130,11 @@ public override Task ExecuteAsync() using Stream destination = _fileSystem.CreateFile(destinationPath); source.CopyTo(destination); } + + if (!OperatingSystem.IsWindows()) + { + _fileSystem.SetUnixFileMode(destinationPath, InfrastructureContent.GetUnixFileMode(relativePath)); + } } return Task.CompletedTask; diff --git a/src/ImageBuilder/FileSystem.cs b/src/ImageBuilder/FileSystem.cs index c807b9839..ce98c726e 100644 --- a/src/ImageBuilder/FileSystem.cs +++ b/src/ImageBuilder/FileSystem.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -29,6 +30,17 @@ public Task WriteAllTextAsync(string path, string? contents, CancellationToken c public Stream CreateFile(string path) => File.Create(path); + /// + public void SetUnixFileMode(string path, UnixFileMode mode) + { + if (OperatingSystem.IsWindows()) + { + throw new PlatformNotSupportedException("Unix file modes are not supported on Windows."); + } + + File.SetUnixFileMode(path, mode); + } + /// public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path); diff --git a/src/ImageBuilder/IFileSystem.cs b/src/ImageBuilder/IFileSystem.cs index 7757d8cdc..29e3ee942 100644 --- a/src/ImageBuilder/IFileSystem.cs +++ b/src/ImageBuilder/IFileSystem.cs @@ -29,6 +29,11 @@ public interface IFileSystem /// Stream CreateFile(string path); + /// + /// Sets the Unix file mode for the specified file. + /// + void SetUnixFileMode(string path, UnixFileMode mode); + /// /// Opens a binary file, reads the contents into a byte array, and then closes the file. /// diff --git a/src/Infrastructure/ContentFileModes.txt b/src/Infrastructure/ContentFileModes.txt new file mode 100644 index 000000000..4bde647c2 --- /dev/null +++ b/src/Infrastructure/ContentFileModes.txt @@ -0,0 +1,80 @@ +0644 CHANGELOG.md +0644 DEV-GUIDE.md +0644 Dockerfile.WithRepo +0644 Dockerfile.syft +0644 Get-BaseImageStatus.ps1 +0644 Get-ImageBuilder.ps1 +0644 Get-ImageNameVars.ps1 +0644 Install-DotNetSdk.ps1 +0644 Invoke-CleanupDocker.ps1 +0644 Invoke-ImageBuilder.ps1 +0644 Invoke-WithRetry.ps1 +0644 Pull-Image.ps1 +0644 Retain-Build.ps1 +0644 Update-ImageBuilder.ps1 +0644 build.ps1 +0644 readme.md +0644 skill-helpers/AzureDevOps.ps1 +0755 skill-helpers/Get-BuildLog.ps1 +0755 skill-helpers/Get-FailingPipelines.ps1 +0644 skill-helpers/Get-RecentBuilds.ps1 +0755 skill-helpers/Show-BuildTimeline.ps1 +0644 skill-helpers/Show-PullRequestBuilds.ps1 +0755 skill-helpers/Show-PullRequestComments.ps1 +0644 templates/1es-official.yml +0644 templates/1es-unofficial.yml +0644 templates/1es.yml +0644 templates/jobs/build-images.yml +0644 templates/jobs/cg-build-projects.yml +0644 templates/jobs/copy-base-images-staging.yml +0644 templates/jobs/copy-base-images.yml +0644 templates/jobs/generate-matrix.yml +0644 templates/jobs/post-build.yml +0644 templates/jobs/publish.yml +0644 templates/jobs/sign-images.yml +0644 templates/jobs/test-images-linux-client.yml +0644 templates/jobs/test-images-windows-client.yml +0644 templates/stages/build-and-test.yml +0644 templates/stages/dotnet/build-and-test.yml +0644 templates/stages/dotnet/build-test-publish-repo.yml +0644 templates/stages/dotnet/publish-config-nonprod.yml +0644 templates/stages/dotnet/publish-config-prod.yml +0644 templates/stages/dotnet/publish.yml +0644 templates/stages/publish.yml +0644 templates/steps/annotate-eol-digests.yml +0644 templates/steps/clean-acr-images.yml +0644 templates/steps/cleanup-docker-linux.yml +0644 templates/steps/cleanup-docker-windows.yml +0644 templates/steps/copy-base-images.yml +0644 templates/steps/download-build-artifact.yml +0644 templates/steps/generate-appsettings.yml +0644 templates/steps/init-common.yml +0644 templates/steps/init-imagebuilder.yml +0644 templates/steps/init-signing-linux.yml +0644 templates/steps/init-testrunner.yml +0644 templates/steps/parse-test-arg-arrays.yml +0644 templates/steps/publish-artifact.yml +0644 templates/steps/publish-readmes.yml +0644 templates/steps/reference-service-connections.yml +0644 templates/steps/retain-build.yml +0644 templates/steps/run-imagebuilder.yml +0644 templates/steps/run-pwsh-with-auth.yml +0644 templates/steps/set-dry-run.yml +0644 templates/steps/set-image-info-path-var.yml +0644 templates/steps/test-images-linux-client.yml +0644 templates/steps/test-images-windows-client.yml +0644 templates/steps/validate-branch.yml +0644 templates/steps/wait-for-mcr-doc-ingestion.yml +0644 templates/steps/wait-for-mcr-image-ingestion.yml +0644 templates/task-prefix-decorator.yml +0644 templates/variables/common-paths.yml +0644 templates/variables/common.yml +0644 templates/variables/dnceng-build-pools.yml +0644 templates/variables/dnceng-project-names.yml +0644 templates/variables/dnceng-signing.yml +0644 templates/variables/docker-images.yml +0644 templates/variables/dotnet/build-test-publish.yml +0644 templates/variables/dotnet/common.yml +0644 templates/variables/dotnet/secrets-unofficial.yml +0644 templates/variables/dotnet/secrets.yml +0644 templates/variables/sdl-pool.yml diff --git a/src/Infrastructure/Generate-ContentFileModes.ps1 b/src/Infrastructure/Generate-ContentFileModes.ps1 new file mode 100644 index 000000000..884047855 --- /dev/null +++ b/src/Infrastructure/Generate-ContentFileModes.ps1 @@ -0,0 +1,45 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +[CmdletBinding()] +param( + [switch]$Check +) + +$ErrorActionPreference = 'Stop' +$contentPath = Join-Path $PSScriptRoot 'Content' +$repoRoot = git -C $PSScriptRoot rev-parse --show-toplevel + +if ($LASTEXITCODE -ne 0) { + throw 'Unable to locate the Git repository root.' +} + +$contentRepoPath = [IO.Path]::GetRelativePath($repoRoot, $contentPath).Replace('\', '/') +$entries = git -C $repoRoot ls-files --stage -- "$contentRepoPath" + +if ($LASTEXITCODE -ne 0) { + throw 'Unable to read infrastructure file modes from the Git index.' +} + +$fileModes = foreach ($entry in $entries) { + if ($entry -notmatch '^(?\d+) \S+ \d+\t(?.+)$') { + throw "Unexpected Git index entry: '$entry'" + } + + $relativePath = $Matches.path.Substring($contentRepoPath.Length + 1) + "$($Matches.mode.Substring(2)) $relativePath" +} + +$outputPath = Join-Path $PSScriptRoot 'ContentFileModes.txt' +$expectedContent = ($fileModes -join "`n") + "`n" + +if ($Check) { + $actualContent = [IO.File]::ReadAllText($outputPath).Replace("`r`n", "`n") + if ($actualContent -ne $expectedContent) { + throw "The generated file mode metadata is stale. Run '$PSCommandPath' to update it." + } + + return +} + +[IO.File]::WriteAllText($outputPath, $expectedContent, [Text.UTF8Encoding]::new($false)) diff --git a/src/Infrastructure/InfrastructureContent.cs b/src/Infrastructure/InfrastructureContent.cs index ff5b6fcc7..fd0a1c77f 100644 --- a/src/Infrastructure/InfrastructureContent.cs +++ b/src/Infrastructure/InfrastructureContent.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Reflection; namespace Microsoft.DotNet.DockerTools.Infrastructure; @@ -23,6 +24,7 @@ public static class InfrastructureContent /// Prefix applied to the LogicalName of every embedded content resource. /// private const string ResourcePrefix = "Content/"; + private const string FileModesResourceName = "ContentFileModes.txt"; private static readonly Assembly s_assembly = typeof(InfrastructureContent).Assembly; @@ -32,12 +34,26 @@ public static class InfrastructureContent /// private static readonly IReadOnlyDictionary s_resourceNamesByPath = BuildIndex(); + private static readonly IReadOnlyDictionary s_fileModesByPath = BuildFileModes(); + /// /// Gets the paths of all embedded content files, relative to the content root and using /// '/' as the directory separator (for example, templates/jobs/build-images.yml). /// public static IReadOnlyList GetRelativePaths() => [.. s_resourceNamesByPath.Keys]; + /// + /// Gets the Unix file mode captured for an embedded content file. + /// + public static UnixFileMode GetUnixFileMode(string relativePath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(relativePath); + + return s_fileModesByPath.TryGetValue(relativePath, out UnixFileMode mode) + ? mode + : throw new KeyNotFoundException($"No embedded infrastructure content found for path '{relativePath}'."); + } + /// /// Opens a stream over the raw bytes of an embedded content file. The caller owns the returned /// stream and is responsible for disposing it. Returning a stream avoids buffering whole files @@ -83,4 +99,34 @@ private static Dictionary BuildIndex() return resourceNamesByPath; } + + private static Dictionary BuildFileModes() + { + using Stream stream = s_assembly.GetManifestResourceStream(FileModesResourceName) + ?? throw new InvalidOperationException($"Embedded resource '{FileModesResourceName}' could not be opened."); + using StreamReader reader = new(stream); + Dictionary fileModesByPath = new(StringComparer.Ordinal); + + while (reader.ReadLine() is { } line) + { + int separatorIndex = line.IndexOf(' '); + if (separatorIndex <= 0 || separatorIndex == line.Length - 1) + { + throw new InvalidOperationException($"Invalid infrastructure file mode entry '{line}'."); + } + + string mode = line[..separatorIndex]; + string relativePath = line[(separatorIndex + 1)..]; + fileModesByPath.Add(relativePath, (UnixFileMode)Convert.ToInt32(mode, 8)); + } + + if (fileModesByPath.Count != s_resourceNamesByPath.Count + || !fileModesByPath.Keys.All(s_resourceNamesByPath.ContainsKey)) + { + throw new InvalidOperationException( + "Embedded infrastructure content and file mode metadata do not contain the same paths."); + } + + return fileModesByPath; + } } diff --git a/src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj b/src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj index 2cf0fb2b0..48cb307fd 100644 --- a/src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj +++ b/src/Infrastructure/Microsoft.DotNet.DockerTools.Infrastructure.csproj @@ -27,7 +27,15 @@ Content/%(RecursiveDir)%(Filename)%(Extension) + + + + + diff --git a/src/Infrastructure/README.md b/src/Infrastructure/README.md index 349f329cf..24434a9b4 100644 --- a/src/Infrastructure/README.md +++ b/src/Infrastructure/README.md @@ -19,6 +19,13 @@ See [issue #2130](https://github.com/dotnet/docker-tools/issues/2130) for backgr The embedded copy lives in `Content/`, under `src/`, because the ImageBuilder Docker build context is `src/`, so `eng/docker-tools/` is not reachable from the container build. +`ContentFileModes.txt` records the Unix modes from Git's index because .NET embedded resources +do not carry filesystem metadata. After changing a file's executable bit, regenerate it with: + +```pwsh +./src/Infrastructure/Generate-ContentFileModes.ps1 +``` + ## Relationship to `eng/docker-tools/` `Content/` is the copy that ships inside ImageBuilder, so infrastructure changes are made here.