diff --git a/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs b/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs index 10e51d53666d..430fdee05751 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs @@ -55,13 +55,13 @@ internal ImageBuilder(ManifestV2 manifest, string manifestMediaType, ImageConfig /// /// Builds the image configuration ready for further processing. /// - internal BuiltImage Build() + internal BuiltImage Build(DateTime? createdAt = null) { // before we build, we need to make sure that any image customizations occur AssignUserFromEnvironment(); AssignPortsFromEnvironment(); - string imageJsonStr = _baseImageConfig.BuildConfig(); + string imageJsonStr = _baseImageConfig.BuildConfig(createdAt ?? DateTime.UtcNow); string imageSha = DigestUtils.ComputeSha256(imageJsonStr); string imageDigest = DigestUtils.FormatSha256Digest(imageSha); long imageSize = Encoding.UTF8.GetBytes(imageJsonStr).Length; diff --git a/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs b/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs index fb2f5e0dd265..417def0b7476 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs @@ -85,7 +85,7 @@ private string GetArchitecture() /// /// Builds in additional configuration and returns updated image configuration in JSON format as string. /// - internal string BuildConfig() + internal string BuildConfig(DateTime createdAt) { var newConfig = new JsonObject(); @@ -141,7 +141,7 @@ internal string BuildConfig() int numberOfLayers = _rootFsLayers.Count; int numberOfNonEmptyLayerHistoryEntries = _history.Count(h => h.empty_layer is null or false); int missingHistoryEntries = numberOfLayers - numberOfNonEmptyLayerHistoryEntries; - HistoryEntry customHistoryEntry = new(created: DateTime.UtcNow, author: ".NET SDK", + HistoryEntry customHistoryEntry = new(created: createdAt, author: ".NET SDK", created_by: $".NET SDK Container Tooling, version {Constants.Version}"); for (int i = 0; i < missingHistoryEntries; i++) { @@ -152,7 +152,7 @@ internal string BuildConfig() { ["config"] = newConfig, //update creation date - ["created"] = RFC3339Format(DateTime.UtcNow), + ["created"] = RFC3339Format(createdAt), ["rootfs"] = new JsonObject() { ["type"] = "layers", diff --git a/src/Containers/Microsoft.NET.Build.Containers/Layer.cs b/src/Containers/Microsoft.NET.Build.Containers/Layer.cs index 1cb57628f390..3198360ba96e 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Layer.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Layer.cs @@ -53,7 +53,17 @@ public static Layer FromDescriptor(Descriptor descriptor) } public static Layer FromDirectory(string directory, string containerPath, bool isWindowsLayer, string manifestMediaType, int? userId = null) + => FromDirectory(directory, containerPath, isWindowsLayer, manifestMediaType, userId, modificationTime: null); + + internal static Layer FromDirectory( + string directory, + string containerPath, + bool isWindowsLayer, + string manifestMediaType, + int? userId, + DateTimeOffset? modificationTime) { + DateTimeOffset entryModificationTime = modificationTime ?? DateTimeOffset.UtcNow; long fileSize; Span hash = stackalloc byte[SHA256.HashSizeInBytes]; Span uncompressedHash = stackalloc byte[SHA256.HashSizeInBytes]; @@ -91,19 +101,22 @@ public static Layer FromDirectory(string directory, string containerPath, bool i string tempTarballPath = ContentStore.GetTempFile(); using (FileStream fs = File.Create(tempTarballPath)) { - using (HashDigestGZipStream gz = new(fs, leaveOpen: true)) + using (LayerTarGZipStream layerStream = new(fs, leaveOpen: true)) { - using (TarWriter writer = new(gz, TarEntryFormat.Pax, leaveOpen: true)) + using (TarWriter writer = new(layerStream, TarEntryFormat.Pax, leaveOpen: true)) { // Windows layers need a Files folder if (isWindowsLayer) { - var entry = new PaxTarEntry(TarEntryType.Directory, "Files", entryAttributes); - writer.WriteEntry(entry); + var entry = new PaxTarEntry(TarEntryType.Directory, "Files", entryAttributes) + { + ModificationTime = entryModificationTime + }; + WriteEntry(writer, layerStream, entry); } // Write an entry for the application directory. - WriteTarEntryForFile(writer, new DirectoryInfo(directory), containerPath, entryAttributes, isWindowsLayer ? null : userId); + WriteTarEntryForFile(writer, layerStream, new DirectoryInfo(directory), containerPath, entryAttributes, isWindowsLayer ? null : userId, entryModificationTime); // Write entries for the application directory contents. var fileList = new FileSystemEnumerable<(FileSystemInfo file, string containerPath)>( @@ -124,21 +137,26 @@ public static Layer FromDirectory(string directory, string containerPath, bool i AttributesToSkip = FileAttributes.System, // Include hidden files RecurseSubdirectories = true }); - foreach (var item in fileList) + // The enumeration order of a directory is filesystem-defined, so it is sorted to keep + // the order of entries in the tar stream stable across machines and builds. + foreach (var item in fileList.OrderBy(static item => item.containerPath, StringComparer.Ordinal)) { - WriteTarEntryForFile(writer, item.file, item.containerPath, entryAttributes, isWindowsLayer ? null : userId); + WriteTarEntryForFile(writer, layerStream, item.file, item.containerPath, entryAttributes, isWindowsLayer ? null : userId, entryModificationTime); } // Windows layers need a Hives folder, we do not need to create any Registry Hive deltas inside if (isWindowsLayer) { - var entry = new PaxTarEntry(TarEntryType.Directory, "Hives", entryAttributes); - writer.WriteEntry(entry); + var entry = new PaxTarEntry(TarEntryType.Directory, "Hives", entryAttributes) + { + ModificationTime = entryModificationTime + }; + WriteEntry(writer, layerStream, entry); } } // Dispose of the TarWriter before getting the hash so the final data get written to the tar stream - int bytesWritten = gz.GetCurrentUncompressedHash(uncompressedHash); + int bytesWritten = layerStream.GetCurrentUncompressedHash(uncompressedHash); Debug.Assert(bytesWritten == uncompressedHash.Length); } @@ -149,8 +167,14 @@ public static Layer FromDirectory(string directory, string containerPath, bool i int bW = SHA256.HashData(fs, hash); Debug.Assert(bW == hash.Length); + static void WriteEntry(TarWriter writer, LayerTarGZipStream layerStream, PaxTarEntry entry) + { + layerStream.NormalizeNextHeader(); + writer.WriteEntry(entry); + } + // Writes a tar entry corresponding to the file system item. - static void WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string containerPath, IEnumerable> entryAttributes, int? userId) + static void WriteTarEntryForFile(TarWriter writer, LayerTarGZipStream layerStream, FileSystemInfo file, string containerPath, IEnumerable> entryAttributes, int? userId, DateTimeOffset modificationTime) { UnixFileMode mode = DetermineFileMode(file); PaxTarEntry entry; @@ -169,12 +193,13 @@ static void WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string c } entry.Mode = mode; + entry.ModificationTime = modificationTime; if (userId is int uid) { entry.Uid = uid; } - writer.WriteEntry(entry); + WriteEntry(writer, layerStream, entry); if (entry.DataStream is not null) { @@ -229,14 +254,27 @@ static UnixFileMode DetermineFileMode(FileSystemInfo file) private static readonly char[] PathSeparators = new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; /// - /// A stream capable of computing the hash digest of raw uncompressed data while also compressing it. + /// Normalizes pax headers while computing the uncompressed tar hash and writing its gzip stream. /// - private sealed class HashDigestGZipStream : Stream + private sealed class LayerTarGZipStream : Stream { + private const int TarBlockSize = 512; + private const int NameLength = 100; + private const int ChecksumOffset = 148; + private const int ChecksumLength = 8; + private const int TypeFlagOffset = 156; + private const byte ExtendedHeaderTypeFlag = (byte)'x'; + + private static ReadOnlySpan NormalizedPaxHeaderName => "./PaxHeaders/."u8; + private readonly IncrementalHash sha256Hash; private readonly GZipStream compressionStream; + private readonly byte[] headerBlock = new byte[TarBlockSize]; + + private int headerBytes; + private bool normalizeNextHeader; - public HashDigestGZipStream(Stream writeStream, bool leaveOpen) + public LayerTarGZipStream(Stream writeStream, bool leaveOpen) { sha256Hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); compressionStream = new GZipStream(writeStream, CompressionMode.Compress, leaveOpen); @@ -244,18 +282,75 @@ public HashDigestGZipStream(Stream writeStream, bool leaveOpen) public override bool CanWrite => true; - public override void Write(byte[] buffer, int offset, int count) + internal void NormalizeNextHeader() { - sha256Hash.AppendData(buffer, offset, count); - compressionStream.Write(buffer, offset, count); + if (normalizeNextHeader || headerBytes != 0) + { + throw new InvalidOperationException("The previous pax header has not been completely written."); + } + + normalizeNextHeader = true; } + public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + public override void Write(ReadOnlySpan buffer) + { + while (normalizeNextHeader && !buffer.IsEmpty) + { + int take = Math.Min(TarBlockSize - headerBytes, buffer.Length); + buffer[..take].CopyTo(headerBlock.AsSpan(headerBytes)); + headerBytes += take; + buffer = buffer[take..]; + + if (headerBytes == TarBlockSize) + { + NormalizePaxHeader(headerBlock); + WriteCore(headerBlock); + headerBytes = 0; + normalizeNextHeader = false; + } + } + + WriteCore(buffer); + } + + private void WriteCore(ReadOnlySpan buffer) { sha256Hash.AppendData(buffer); compressionStream.Write(buffer); } + private static void NormalizePaxHeader(Span header) + { + if (header[TypeFlagOffset] != ExtendedHeaderTypeFlag) + { + return; + } + + Span name = header[..NameLength]; + name.Clear(); + NormalizedPaxHeaderName.CopyTo(name); + + Span checksumField = header.Slice(ChecksumOffset, ChecksumLength); + checksumField.Fill((byte)' '); + + int checksum = 0; + foreach (byte b in header) + { + checksum += b; + } + + for (int i = 5; i >= 0; i--) + { + checksumField[i] = (byte)('0' + (checksum & 7)); + checksum >>= 3; + } + + checksumField[6] = 0; + checksumField[7] = (byte)' '; + } + public override void Flush() { compressionStream.Flush(); @@ -267,11 +362,17 @@ protected override void Dispose(bool disposing) { try { - sha256Hash.Dispose(); + if (headerBytes > 0) + { + WriteCore(headerBlock.AsSpan(0, headerBytes)); + headerBytes = 0; + } + compressionStream.Dispose(); } finally { + sha256Hash.Dispose(); base.Dispose(disposing); } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net11.0/PublicAPI.Unshipped.txt b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net11.0/PublicAPI.Unshipped.txt index 5db830dd4f46..55526886cd07 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net11.0/PublicAPI.Unshipped.txt +++ b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net11.0/PublicAPI.Unshipped.txt @@ -203,6 +203,8 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.g Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SourceDateEpoch.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SourceDateEpoch.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.CreateNewImage() -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Dispose() -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Entrypoint.get -> Microsoft.Build.Framework.ITaskItem![]! @@ -249,6 +251,8 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.get -> stri Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.get -> bool Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateCreatedLabels.get -> bool +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateCreatedLabels.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.get -> bool Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.get -> bool diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs index d33273ee309f..4c47a0bd6e57 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs @@ -142,12 +142,23 @@ partial class CreateNewImage /// public string ContainerUser { get; set; } + /// + /// The Unix timestamp used to make generated container artifacts reproducible. + /// + public string SourceDateEpoch { get; set; } + /// /// If true, the tooling may create labels on the generated images. /// [Required] public bool GenerateLabels { get; set; } + /// + /// If true, the tooling will generate the OCI image and artifact creation labels. + /// + [Required] + public bool GenerateCreatedLabels { get; set; } + /// /// If true, the tooling will generate an org.opencontainers.image.base.digest label on the generated images containing the digest of the chosen base image. /// @@ -212,6 +223,7 @@ public CreateNewImage() RuntimeIdentifierGraphPath = ""; LocalRegistry = ""; ContainerUser = ""; + SourceDateEpoch = ""; GeneratedContainerConfiguration = ""; GeneratedContainerManifest = ""; @@ -222,6 +234,7 @@ public CreateNewImage() GeneratedDigestLabel = null; GenerateLabels = false; + GenerateCreatedLabels = false; GenerateDigestLabel = false; TaskResources = Resource.Manager; diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs index 64506e88b352..073fbc63c405 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Globalization; using Microsoft.Build.Framework; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.MSBuild; @@ -17,6 +18,23 @@ public sealed partial class CreateNewImage : Microsoft.Build.Utilities.Task, ICa public void Cancel() => _cancellationTokenSource.Cancel(); + internal static DateTime? ParseSourceDateEpoch(string? value) + { + if (!long.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out long seconds) || seconds < 0) + { + return null; + } + + try + { + return DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime; + } + catch (ArgumentOutOfRangeException) + { + return null; + } + } + public override bool Execute() { try @@ -163,8 +181,15 @@ private async Task ExecuteAsyncCore(ILogger logger, ILoggerFactory msbuild imageBuilder.ManifestMediaType, requestedImageFormat, destinationImageReference); + DateTime createdAt = ParseSourceDateEpoch(SourceDateEpoch) ?? DateTime.UtcNow; var userId = imageBuilder.IsWindows ? null : ContainerHelpers.TryParseUserId(ContainerUser); - Layer newLayer = Layer.FromDirectory(PublishDirectory, WorkingDirectory, imageBuilder.IsWindows, imageBuilder.ManifestMediaType, userId); + Layer newLayer = Layer.FromDirectory( + PublishDirectory, + WorkingDirectory, + imageBuilder.IsWindows, + imageBuilder.ManifestMediaType, + userId, + modificationTime: createdAt); imageBuilder.AddLayer(newLayer); imageBuilder.SetWorkingDirectory(WorkingDirectory); @@ -180,6 +205,13 @@ private async Task ExecuteAsyncCore(ILogger logger, ILoggerFactory msbuild imageBuilder.AddLabel(label.ItemSpec, label.GetMetadata("Value")); } + if (GenerateCreatedLabels) + { + string createdLabel = createdAt.ToString("o", CultureInfo.InvariantCulture); + imageBuilder.AddLabel("org.opencontainers.image.created", createdLabel); + imageBuilder.AddLabel("org.opencontainers.artifact.created", createdLabel); + } + if (GenerateDigestLabel) { (baseImageLabel, baseImageDigest) = imageBuilder.AddBaseImageDigestLabel(); @@ -208,7 +240,7 @@ private async Task ExecuteAsyncCore(ILogger logger, ILoggerFactory msbuild return false; } - BuiltImage builtImage = imageBuilder.Build(); + BuiltImage builtImage = imageBuilder.Build(createdAt); cancellationToken.ThrowIfCancellationRequested(); // at this point we're done with modifications and are just pushing the data other places diff --git a/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets b/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets index 07f09c6aa9b8..bdf847b32305 100644 --- a/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets +++ b/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets @@ -149,7 +149,6 @@ - @@ -275,9 +274,11 @@ ContainerEnvironmentVariables="@(ContainerEnvironmentVariables)" ContainerRuntimeIdentifier="$(ContainerRuntimeIdentifier)" ContainerUser="$(ContainerUser)" + SourceDateEpoch="$(SOURCE_DATE_EPOCH)" RuntimeIdentifierGraphPath="$(RuntimeIdentifierGraphPath)" SkipPublishing="$(_SkipContainerPublishing)" GenerateLabels="$(ContainerGenerateLabels)" + GenerateCreatedLabels="$(ContainerGenerateLabelsImageCreated)" GenerateDigestLabel="$(ContainerGenerateLabelsImageBaseDigest)"> @@ -332,6 +333,7 @@ _ContainerPort=@(ContainerPort->'%(Identity):%(Type)'); _ContainerEnvironmentVariables=@(ContainerEnvironmentVariable->'%(Identity):%(Value)'); ContainerGenerateLabels=$(ContainerGenerateLabels); + ContainerGenerateLabelsImageCreated=$(ContainerGenerateLabelsImageCreated); ContainerGenerateLabelsImageBaseDigest=$(ContainerGenerateLabelsImageBaseDigest); _SkipContainerPublishing=$(_SkipContainerPublishing); ContainerImageFormat=$(_SingleImageContainerFormat); diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/CreateNewImageTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/CreateNewImageTests.cs index 8de0b7c5c1ae..ed1a99ea2783 100644 --- a/test/Microsoft.NET.Build.Containers.UnitTests/CreateNewImageTests.cs +++ b/test/Microsoft.NET.Build.Containers.UnitTests/CreateNewImageTests.cs @@ -11,6 +11,34 @@ namespace Microsoft.NET.Build.Containers.UnitTests; [TestClass] public class CreateNewImageTests { + [TestMethod] + [DataRow("0", 0L)] + [DataRow("1636374896", 1636374896L)] + [DataRow("99999999999", 99999999999L)] + [DataRow("100000000000", 100000000000L)] + public void ParseSourceDateEpochReturnsUtcTimestamp(string value, long expectedSeconds) + { + DateTime? actual = CreateNewImage.ParseSourceDateEpoch(value); + + Assert.AreEqual(DateTimeOffset.FromUnixTimeSeconds(expectedSeconds).UtcDateTime, actual); + Assert.AreEqual(DateTimeKind.Utc, actual!.Value.Kind); + } + + [TestMethod] + [DataRow(null, DisplayName = "unset")] + [DataRow("", DisplayName = "empty")] + [DataRow(" ", DisplayName = "whitespace")] + [DataRow(" 1636374896\n", DisplayName = "surrounding whitespace")] + [DataRow("not-a-number", DisplayName = "not numeric")] + [DataRow("1636374896.5", DisplayName = "fractional")] + [DataRow("-1", DisplayName = "negative")] + [DataRow("0x10", DisplayName = "hexadecimal")] + [DataRow("1,636,374,896", DisplayName = "group separators")] + [DataRow("99999999999999999999", DisplayName = "larger than Int64")] + [DataRow("253402300800", DisplayName = "outside DateTimeOffset's range")] + public void ParseSourceDateEpochReturnsNullForInvalidValues(string? value) + => Assert.IsNull(CreateNewImage.ParseSourceDateEpoch(value)); + [TestMethod] // Entrypoint, backwards compatibility. [DataRow("", "entrypointArg", "appCommand", "", "", null, new[] { "appCommand" }, new[] { "entrypointArg" })] diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs index 849826edab2f..5c2d4e523b0c 100644 --- a/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs +++ b/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs @@ -82,7 +82,7 @@ public void CanAddLabelsToImage() baseConfig.AddLabel("testLabel1", "v1"); baseConfig.AddLabel("testLabel2", "v2"); - string readyImage = baseConfig.BuildConfig(); + string readyImage = baseConfig.BuildConfig(DateTime.UtcNow); JsonNode? result = JsonNode.Parse(readyImage); @@ -153,7 +153,7 @@ public void CanPreserveExistingLabels() baseConfig.AddLabel("testLabel1", "v1"); baseConfig.AddLabel("existing2", "v2"); - string readyImage = baseConfig.BuildConfig(); + string readyImage = baseConfig.BuildConfig(DateTime.UtcNow); JsonNode? result = JsonNode.Parse(readyImage); @@ -221,7 +221,7 @@ public void CanAddPortsToImage() baseConfig.ExposePort(6000, PortType.tcp); baseConfig.ExposePort(6010, PortType.udp); - string readyImage = baseConfig.BuildConfig(); + string readyImage = baseConfig.BuildConfig(DateTime.UtcNow); JsonNode? result = JsonNode.Parse(readyImage); @@ -295,7 +295,7 @@ public void CanPreserveExistingPorts() baseConfig.ExposePort(6100, PortType.udp); baseConfig.ExposePort(6200, PortType.tcp); - string readyImage = baseConfig.BuildConfig(); + string readyImage = baseConfig.BuildConfig(DateTime.UtcNow); JsonNode? result = JsonNode.Parse(readyImage); @@ -379,7 +379,7 @@ public void HistoryEntriesMatchNonEmptyLayers() ImageConfig baseConfig = new(node); - string readyImage = baseConfig.BuildConfig(); + string readyImage = baseConfig.BuildConfig(DateTime.UtcNow); JsonNode? result = JsonNode.Parse(readyImage); diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs index da884160594a..106d6068f3a2 100644 --- a/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs +++ b/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs @@ -51,8 +51,21 @@ public class ImageConfigTests public void PassesThroughPropertyEvenThoughPropertyIsntExplicitlyHandled(string property) { ImageConfig c = new(SampleImageConfig); - JsonNode after = JsonNode.Parse(c.BuildConfig())!; + JsonNode after = JsonNode.Parse(c.BuildConfig(DateTime.UtcNow))!; JsonNode? prop = after["config"]?[property]; Assert.IsNotNull(prop); } + + [TestMethod] + public void BuildConfigUsesProvidedCreationTime() + { + var createdAt = new DateTime(2021, 11, 8, 12, 34, 56, DateTimeKind.Utc); + ImageConfig config = new(SampleImageConfig); + + JsonNode result = JsonNode.Parse(config.BuildConfig(createdAt))!; + + Assert.AreEqual("2021-11-08T12:34:56.0000000Z", result["created"]?.GetValue()); + Assert.IsTrue(result["history"]!.AsArray().All(entry => + entry?["created"]?.GetValue() == "2021-11-08T12:34:56.0000000Z")); + } } diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/LayerReproducibilityTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/LayerReproducibilityTests.cs new file mode 100644 index 000000000000..7be5aebc82ac --- /dev/null +++ b/test/Microsoft.NET.Build.Containers.UnitTests/LayerReproducibilityTests.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Formats.Tar; +using System.IO.Compression; + +namespace Microsoft.NET.Build.Containers.UnitTests; + +[TestClass] +public class LayerReproducibilityTests +{ + private const string ManifestMediaType = "application/vnd.docker.distribution.manifest.v2+json"; + private static readonly DateTimeOffset ReproducibleTimestamp = DateTimeOffset.FromUnixTimeSeconds(1636374896); + + private string CreateContentDirectory() + { + string directory = Path.Combine(TestContext.ResultsDirectory!, Path.GetRandomFileName()); + Directory.CreateDirectory(Path.Combine(directory, "subdirectory")); + File.WriteAllText(Path.Combine(directory, "app.dll"), $"some content for {TestContext.TestName}"); + File.WriteAllText(Path.Combine(directory, "subdirectory", "app.deps.json"), $"some other content for {TestContext.TestName}"); + return directory; + } + + public TestContext TestContext { get; set; } = null!; + + [TestMethod] + public void LayersBuiltFromIdenticalContentHaveTheSameDigest() + { + // This is the behavior the change exists for: publishing the same content twice should produce + // the same layer, so a rebuild does not appear to downstream tooling as a new artifact. + string first = CreateContentDirectory(); + string second = CreateContentDirectory(); + File.SetLastWriteTimeUtc(Path.Combine(second, "app.dll"), new DateTime(2001, 2, 3, 4, 5, 6, DateTimeKind.Utc)); + + Layer firstLayer = Layer.FromDirectory(first, "/app", false, ManifestMediaType, userId: null, modificationTime: ReproducibleTimestamp); + Layer secondLayer = Layer.FromDirectory(second, "/app", false, ManifestMediaType, userId: null, modificationTime: ReproducibleTimestamp); + + Assert.AreEqual(firstLayer.Descriptor.Digest, secondLayer.Descriptor.Digest); + Assert.AreEqual(firstLayer.Descriptor.Size, secondLayer.Descriptor.Size); + + // The layer must not depend on the process that produced it, which the process id in the + // pax extended header names would otherwise leak in. + using FileStream compressed = File.OpenRead(firstLayer.BackingFile); + using var decompressed = new GZipStream(compressed, CompressionMode.Decompress); + using var text = new StreamReader(decompressed); + Assert.IsFalse( + text.ReadToEnd().Contains($"PaxHeaders.{Environment.ProcessId}", StringComparison.Ordinal), + "The layer should not contain the current process id."); + } + + [TestMethod] + public void LayersBuiltFromDifferentContentHaveDifferentDigests() + { + // The digest must still be a function of the content: pinning the timestamp must not make + // genuinely different inputs collide. + string first = CreateContentDirectory(); + string second = CreateContentDirectory(); + File.WriteAllText(Path.Combine(second, "app.dll"), "some different content"); + + Assert.AreNotEqual( + Layer.FromDirectory(first, "/app", false, ManifestMediaType, userId: null, modificationTime: ReproducibleTimestamp).Descriptor.Digest, + Layer.FromDirectory(second, "/app", false, ManifestMediaType, userId: null, modificationTime: ReproducibleTimestamp).Descriptor.Digest); + } + + [TestMethod] + public void EveryLayerEntryCarriesTheTimestampFromSourceDateEpoch() + { + Layer layer = Layer.FromDirectory( + CreateContentDirectory(), + "/app", + false, + ManifestMediaType, + userId: null, + modificationTime: ReproducibleTimestamp); + + using FileStream compressed = File.OpenRead(layer.BackingFile); + using var decompressed = new GZipStream(compressed, CompressionMode.Decompress); + using var reader = new TarReader(decompressed); + + int entries = 0; + while (reader.GetNextEntry() is TarEntry entry) + { + entries++; + Assert.AreEqual(ReproducibleTimestamp, entry.ModificationTime, $"Entry '{entry.Name}' has an unexpected timestamp."); + } + + Assert.AreEqual(4, entries, "Expected the app directory, the subdirectory and the two files."); + } + + [TestMethod] + public void LayerPreservesFileContentThatLooksLikeAPaxHeader() + { + byte[] expected = new byte[512]; + expected.AsSpan().Fill((byte)'a'); + "ustar"u8.CopyTo(expected.AsSpan(257)); + expected[156] = (byte)'x'; + "./PaxHeaders.99999/."u8.CopyTo(expected); + + string directory = CreateContentDirectory(); + File.WriteAllBytes(Path.Combine(directory, "app.dll"), expected); + Layer layer = Layer.FromDirectory( + directory, + "/app", + false, + ManifestMediaType, + userId: null, + modificationTime: ReproducibleTimestamp); + + using FileStream compressed = File.OpenRead(layer.BackingFile); + using var decompressed = new GZipStream(compressed, CompressionMode.Decompress); + using var reader = new TarReader(decompressed); + + while (reader.GetNextEntry() is TarEntry entry) + { + if (entry.Name == "app/app.dll") + { + using var actual = new MemoryStream(); + entry.DataStream!.CopyTo(actual); + Assert.AreSequenceEqual(expected, actual.ToArray()); + return; + } + } + + Assert.Fail("The layer did not contain app/app.dll."); + } +}