[release/11.0.1xx-rc1] Source code updates from dotnet/sdk

src/sdk/src/Cli/dotnet/Commands/Pack/PackCommand.cs

@@ -121,6 +121,9 @@ public static int RunPackCommand(ParseResult parseResult)
         if (version != null)
             packArgs.Version = version.ToNormalizedString();
 
+        if (parseResult.GetValue(definition.IncludeSymbolsOption))
+            packArgs.Symbols = true;
+
         var configuration = parseResult.GetValue(definition.ConfigurationOption) ?? "Debug";
         packArgs.Properties["configuration"] = configuration;
 

src/sdk/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs

@@ -55,13 +55,13 @@ internal ImageBuilder(ManifestV2 manifest, string manifestMediaType, ImageConfig
     /// <summary>
     /// Builds the image configuration <see cref="BuiltImage"/> ready for further processing.
     /// </summary>
-    internal BuiltImage Build()
+    internal BuiltImage Build(DateTime? createdAt = null)
     {
         // before we build, we need to make sure that any image customizations occur
         AssignUserFromEnvironment();
         AssignPortsFromEnvironment();
 
-        string imageJsonStr = _baseImageConfig.BuildConfig();
+        string imageJsonStr = _baseImageConfig.BuildConfig(createdAt ?? DateTime.UtcNow);
         string imageSha = DigestUtils.ComputeSha256(imageJsonStr);
         string imageDigest = DigestUtils.FormatSha256Digest(imageSha);
         long imageSize = Encoding.UTF8.GetBytes(imageJsonStr).Length;

src/sdk/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs

@@ -85,7 +85,7 @@ private string GetArchitecture()
     /// <summary>
     /// Builds in additional configuration and returns updated image configuration in JSON format as string.
     /// </summary>
-    internal string BuildConfig()
+    internal string BuildConfig(DateTime createdAt)
     {
         var newConfig = new JsonObject();
 
@@ -141,7 +141,7 @@ internal string BuildConfig()
         int numberOfLayers = _rootFsLayers.Count;
         int numberOfNonEmptyLayerHistoryEntries = _history.Count(h => h.empty_layer is null or false);
         int missingHistoryEntries = numberOfLayers - numberOfNonEmptyLayerHistoryEntries;
-        HistoryEntry customHistoryEntry = new(created: DateTime.UtcNow, author: ".NET SDK",
+        HistoryEntry customHistoryEntry = new(created: createdAt, author: ".NET SDK",
             created_by: $".NET SDK Container Tooling, version {Constants.Version}");
         for (int i = 0; i < missingHistoryEntries; i++)
         {
@@ -152,7 +152,7 @@ internal string BuildConfig()
         {
             ["config"] = newConfig,
             //update creation date
-            ["created"] = RFC3339Format(DateTime.UtcNow),
+            ["created"] = RFC3339Format(createdAt),
             ["rootfs"] = new JsonObject()
             {
                 ["type"] = "layers",

src/sdk/src/Containers/Microsoft.NET.Build.Containers/Layer.cs

@@ -53,7 +53,17 @@ public static Layer FromDescriptor(Descriptor descriptor)
     }
 
     public static Layer FromDirectory(string directory, string containerPath, bool isWindowsLayer, string manifestMediaType, int? userId = null)
+        => FromDirectory(directory, containerPath, isWindowsLayer, manifestMediaType, userId, modificationTime: null);
+
+    internal static Layer FromDirectory(
+        string directory,
+        string containerPath,
+        bool isWindowsLayer,
+        string manifestMediaType,
+        int? userId,
+        DateTimeOffset? modificationTime)
     {
+        DateTimeOffset entryModificationTime = modificationTime ?? DateTimeOffset.UtcNow;
         long fileSize;
         Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
         Span<byte> uncompressedHash = stackalloc byte[SHA256.HashSizeInBytes];
@@ -91,19 +101,22 @@ public static Layer FromDirectory(string directory, string containerPath, bool i
         string tempTarballPath = ContentStore.GetTempFile();
         using (FileStream fs = File.Create(tempTarballPath))
         {
-            using (HashDigestGZipStream gz = new(fs, leaveOpen: true))
+            using (LayerTarGZipStream layerStream = new(fs, leaveOpen: true))
             {
-                using (TarWriter writer = new(gz, TarEntryFormat.Pax, leaveOpen: true))
+                using (TarWriter writer = new(layerStream, TarEntryFormat.Pax, leaveOpen: true))
                 {
                     // Windows layers need a Files folder
                     if (isWindowsLayer)
                     {
-                        var entry = new PaxTarEntry(TarEntryType.Directory, "Files", entryAttributes);
-                        writer.WriteEntry(entry);
+                        var entry = new PaxTarEntry(TarEntryType.Directory, "Files", entryAttributes)
+                        {
+                            ModificationTime = entryModificationTime
+                        };
+                        WriteEntry(writer, layerStream, entry);
                     }
 
                     // Write an entry for the application directory.
-                    WriteTarEntryForFile(writer, new DirectoryInfo(directory), containerPath, entryAttributes, isWindowsLayer ? null : userId);
+                    WriteTarEntryForFile(writer, layerStream, new DirectoryInfo(directory), containerPath, entryAttributes, isWindowsLayer ? null : userId, entryModificationTime);
 
                     // Write entries for the application directory contents.
                     var fileList = new FileSystemEnumerable<(FileSystemInfo file, string containerPath)>(
@@ -124,21 +137,26 @@ public static Layer FromDirectory(string directory, string containerPath, bool i
                                     AttributesToSkip = FileAttributes.System, // Include hidden files
                                     RecurseSubdirectories = true
                                 });
-                    foreach (var item in fileList)
+                    // The enumeration order of a directory is filesystem-defined, so it is sorted to keep
+                    // the order of entries in the tar stream stable across machines and builds.
+                    foreach (var item in fileList.OrderBy(static item => item.containerPath, StringComparer.Ordinal))
                     {
-                        WriteTarEntryForFile(writer, item.file, item.containerPath, entryAttributes, isWindowsLayer ? null : userId);
+                        WriteTarEntryForFile(writer, layerStream, item.file, item.containerPath, entryAttributes, isWindowsLayer ? null : userId, entryModificationTime);
                     }
 
                     // Windows layers need a Hives folder, we do not need to create any Registry Hive deltas inside
                     if (isWindowsLayer)
                     {
-                        var entry = new PaxTarEntry(TarEntryType.Directory, "Hives", entryAttributes);
-                        writer.WriteEntry(entry);
+                        var entry = new PaxTarEntry(TarEntryType.Directory, "Hives", entryAttributes)
+                        {
+                            ModificationTime = entryModificationTime
+                        };
+                        WriteEntry(writer, layerStream, entry);
                     }
 
                 } // Dispose of the TarWriter before getting the hash so the final data get written to the tar stream
 
-                int bytesWritten = gz.GetCurrentUncompressedHash(uncompressedHash);
+                int bytesWritten = layerStream.GetCurrentUncompressedHash(uncompressedHash);
                 Debug.Assert(bytesWritten == uncompressedHash.Length);
             }
 
@@ -149,8 +167,14 @@ public static Layer FromDirectory(string directory, string containerPath, bool i
             int bW = SHA256.HashData(fs, hash);
             Debug.Assert(bW == hash.Length);
 
+            static void WriteEntry(TarWriter writer, LayerTarGZipStream layerStream, PaxTarEntry entry)
+            {
+                layerStream.NormalizeNextHeader();
+                writer.WriteEntry(entry);
+            }
+
             // Writes a tar entry corresponding to the file system item.
-            static void WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string containerPath, IEnumerable<KeyValuePair<string, string>> entryAttributes, int? userId)
+            static void WriteTarEntryForFile(TarWriter writer, LayerTarGZipStream layerStream, FileSystemInfo file, string containerPath, IEnumerable<KeyValuePair<string, string>> entryAttributes, int? userId, DateTimeOffset modificationTime)
             {
                 UnixFileMode mode = DetermineFileMode(file);
                 PaxTarEntry entry;
@@ -169,12 +193,13 @@ static void WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string c
                 }
 
                 entry.Mode = mode;
+                entry.ModificationTime = modificationTime;
                 if (userId is int uid)
                 {
                     entry.Uid = uid;
                 }
 
-                writer.WriteEntry(entry);
+                WriteEntry(writer, layerStream, entry);
 
                 if (entry.DataStream is not null)
                 {
@@ -229,33 +254,103 @@ static UnixFileMode DetermineFileMode(FileSystemInfo file)
     private static readonly char[] PathSeparators = new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar };
 
     /// <summary>
-    /// A stream capable of computing the hash digest of raw uncompressed data while also compressing it.
+    /// Normalizes pax headers while computing the uncompressed tar hash and writing its gzip stream.
     /// </summary>
-    private sealed class HashDigestGZipStream : Stream
+    private sealed class LayerTarGZipStream : Stream
     {
+        private const int TarBlockSize = 512;
+        private const int NameLength = 100;
+        private const int ChecksumOffset = 148;
+        private const int ChecksumLength = 8;
+        private const int TypeFlagOffset = 156;
+        private const byte ExtendedHeaderTypeFlag = (byte)'x';
+
+        private static ReadOnlySpan<byte> NormalizedPaxHeaderName => "./PaxHeaders/."u8;
+
         private readonly IncrementalHash sha256Hash;
         private readonly GZipStream compressionStream;
+        private readonly byte[] headerBlock = new byte[TarBlockSize];
+
+        private int headerBytes;
+        private bool normalizeNextHeader;
 
-        public HashDigestGZipStream(Stream writeStream, bool leaveOpen)
+        public LayerTarGZipStream(Stream writeStream, bool leaveOpen)
         {
             sha256Hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
             compressionStream = new GZipStream(writeStream, CompressionMode.Compress, leaveOpen);
         }
 
         public override bool CanWrite => true;
 
-        public override void Write(byte[] buffer, int offset, int count)
+        internal void NormalizeNextHeader()
         {
-            sha256Hash.AppendData(buffer, offset, count);
-            compressionStream.Write(buffer, offset, count);
+            if (normalizeNextHeader || headerBytes != 0)
+            {
+                throw new InvalidOperationException("The previous pax header has not been completely written.");
+            }
+
+            normalizeNextHeader = true;
         }
 
+        public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count));
+
         public override void Write(ReadOnlySpan<byte> buffer)
+        {
+            while (normalizeNextHeader && !buffer.IsEmpty)
+            {
+                int take = Math.Min(TarBlockSize - headerBytes, buffer.Length);
+                buffer[..take].CopyTo(headerBlock.AsSpan(headerBytes));
+                headerBytes += take;
+                buffer = buffer[take..];
+
+                if (headerBytes == TarBlockSize)
+                {
+                    NormalizePaxHeader(headerBlock);
+                    WriteCore(headerBlock);
+                    headerBytes = 0;
+                    normalizeNextHeader = false;
+                }
+            }
+
+            WriteCore(buffer);
+        }
+
+        private void WriteCore(ReadOnlySpan<byte> buffer)
         {
             sha256Hash.AppendData(buffer);
             compressionStream.Write(buffer);
         }
 
+        private static void NormalizePaxHeader(Span<byte> header)
+        {
+            if (header[TypeFlagOffset] != ExtendedHeaderTypeFlag)
+            {
+                return;
+            }
+
+            Span<byte> name = header[..NameLength];
+            name.Clear();
+            NormalizedPaxHeaderName.CopyTo(name);
+
+            Span<byte> checksumField = header.Slice(ChecksumOffset, ChecksumLength);
+            checksumField.Fill((byte)' ');
+
+            int checksum = 0;
+            foreach (byte b in header)
+            {
+                checksum += b;
+            }
+
+            for (int i = 5; i >= 0; i--)
+            {
+                checksumField[i] = (byte)('0' + (checksum & 7));
+                checksum >>= 3;
+            }
+
+            checksumField[6] = 0;
+            checksumField[7] = (byte)' ';
+        }
+
         public override void Flush()
         {
             compressionStream.Flush();
@@ -267,11 +362,17 @@ protected override void Dispose(bool disposing)
         {
             try
             {
-                sha256Hash.Dispose();
+                if (headerBytes > 0)
+                {
+                    WriteCore(headerBlock.AsSpan(0, headerBytes));
+                    headerBytes = 0;
+                }
+
                 compressionStream.Dispose();
             }
             finally
             {
+                sha256Hash.Dispose();
                 base.Dispose(disposing);
             }
         }

src/sdk/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net11.0/PublicAPI.Unshipped.txt

@@ -203,6 +203,8 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.g
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.get -> string!
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.set -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SourceDateEpoch.get -> string!
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SourceDateEpoch.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.CreateNewImage() -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Dispose() -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Entrypoint.get -> Microsoft.Build.Framework.ITaskItem![]!
@@ -249,6 +251,8 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.get -> stri
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.get -> bool
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.set -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateCreatedLabels.get -> bool
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateCreatedLabels.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.get -> bool
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.get -> bool

src/sdk/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs

@@ -142,12 +142,23 @@ partial class CreateNewImage
     /// </summary>
     public string ContainerUser { get; set; }
 
+    /// <summary>
+    /// The Unix timestamp used to make generated container artifacts reproducible.
+    /// </summary>
+    public string SourceDateEpoch { get; set; }
+
     /// <summary>
     /// If true, the tooling may create labels on the generated images.
     /// </summary>
     [Required]
     public bool GenerateLabels { get; set; }
 
+    /// <summary>
+    /// If true, the tooling will generate the OCI image and artifact creation labels.
+    /// </summary>
+    [Required]
+    public bool GenerateCreatedLabels { get; set; }
+
     /// <summary>
     /// If true, the tooling will generate an <c>org.opencontainers.image.base.digest</c> label on the generated images containing the digest of the chosen base image.
     /// </summary>
@@ -212,6 +223,7 @@ public CreateNewImage()
         RuntimeIdentifierGraphPath = "";
         LocalRegistry = "";
         ContainerUser = "";
+        SourceDateEpoch = "";
 
         GeneratedContainerConfiguration = "";
         GeneratedContainerManifest = "";
@@ -222,6 +234,7 @@ public CreateNewImage()
         GeneratedDigestLabel = null;
 
         GenerateLabels = false;
+        GenerateCreatedLabels = false;
         GenerateDigestLabel = false;
 
         TaskResources = Resource.Manager;