diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c971100db..a5ed0f9a0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,6 +16,16 @@ parameters: displayName: Run Extended Tests? type: boolean default: false + +- name: RunBarValidation + displayName: Register a BAR validation build without promotion? + type: boolean + default: false + +- name: BarValidationSignedBuildId + displayName: Official AndroidX build ID supplying nuget-signed + type: number + default: 0 variables: # Variables used by both AndroidX/GPS go in the template @@ -94,3 +104,8 @@ extends: os: windows - template: build/ci/stage-sign-artifacts.yml@self + + - template: build/ci/stage-publish-signed-artifacts.yml@self + parameters: + runValidation: ${{ parameters.RunBarValidation }} + validationSignedBuildId: ${{ parameters.BarValidationSignedBuildId }} diff --git a/build/ci/stage-publish-signed-artifacts.yml b/build/ci/stage-publish-signed-artifacts.yml new file mode 100644 index 000000000..73befd335 --- /dev/null +++ b/build/ci/stage-publish-signed-artifacts.yml @@ -0,0 +1,223 @@ +# Filters and publishes only real-signed packages that are absent from the public dotnet10 feed. + +parameters: +- name: runValidation + type: boolean + default: false +- name: validationSignedBuildId + type: number + default: 0 + +stages: +- stage: publish_signed_packages + displayName: Publish Signed Packages + dependsOn: sign_artifacts + condition: and(succeeded(), or(and(eq('${{ parameters.runValidation }}', 'true'), eq(variables['Build.Reason'], 'Manual'), ne('${{ parameters.validationSignedBuildId }}', '0')), and(eq(variables['barPublishingEnabled'], 'true'), or(and(eq(variables['Build.SourceBranch'], 'refs/heads/main'), ne(variables['Build.Reason'], 'Schedule')), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')), ne(variables['Build.Reason'], 'PullRequest')))) + lockBehavior: sequential + + jobs: + - job: publish_signed_packages + displayName: Filter, register, and promote signed packages + timeoutInMinutes: 240 + pool: + name: AzurePipelines-EO + image: 1ESPT-Windows2025 + demands: Cmd + os: windows + steps: + - download: none + + - checkout: self + clean: true + fetchDepth: 3 + + - ${{ if eq(parameters.runValidation, true) }}: + - task: PowerShell@2 + displayName: Validate signed package source build + inputs: + targetType: inline + script: | + $ErrorActionPreference = 'Stop' + $buildId = [int]'${{ parameters.validationSignedBuildId }}' + if ($buildId -le 0) { + throw 'BarValidationSignedBuildId must be a positive Azure DevOps build ID.' + } + + $headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" } + $buildUri = "$(System.CollectionUri)$(System.TeamProject)/_apis/build/builds/$buildId`?api-version=7.1" + $build = Invoke-RestMethod -Uri $buildUri -Headers $headers + $isRealSigned = ( + ($build.sourceBranch -eq 'refs/heads/main' -and $build.reason -ne 'schedule') -or + $build.sourceBranch.StartsWith('refs/heads/release/', [StringComparison]::Ordinal) + ) -and $build.reason -ne 'pullRequest' + + if ($build.definition.id -ne [int]'$(System.DefinitionId)' -or + $build.result -ne 'succeeded' -or + -not $isRealSigned) { + throw "Build $buildId is not a successful real-sign AndroidX build." + } + + $artifactsUri = "$(System.CollectionUri)$(System.TeamProject)/_apis/build/builds/$buildId/artifacts?api-version=7.1" + $artifacts = Invoke-RestMethod -Uri $artifactsUri -Headers $headers + $signedArtifact = @($artifacts.value | Where-Object { + $_.name -eq 'nuget-signed' -and $_.resource.type -eq 'PipelineArtifact' + }) + if ($signedArtifact.Count -ne 1) { + throw "Build $buildId must contain exactly one nuget-signed pipeline artifact." + } + + Write-Host "Using real-signed packages from AndroidX build $($build.buildNumber) ($buildId)." + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - task: DownloadPipelineArtifact@2 + displayName: Download validation real-signed NuGet packages + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + buildVersionToDownload: specific + pipelineId: ${{ parameters.validationSignedBuildId }} + artifactName: nuget-signed + targetPath: $(Pipeline.Workspace)\nuget-signed + + - ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: Download real-signed NuGet packages + inputs: + artifactName: nuget-signed + targetPath: $(Pipeline.Workspace)\nuget-signed + + - task: UseDotNet@2 + displayName: Install .NET SDK + inputs: + version: 10.x + + - task: NuGetAuthenticate@1 + displayName: Authenticate to Azure Artifacts + + - task: PowerShell@2 + displayName: Configure scoped Arcade restore + inputs: + targetType: inline + script: | + $toolsetDirectory = '$(System.DefaultWorkingDirectory)\artifacts\toolset' + New-Item -ItemType Directory -Force -Path $toolsetDirectory | Out-Null + Copy-Item ` + '$(System.DefaultWorkingDirectory)\build\publishing\NuGet.config' ` + "$toolsetDirectory\NuGet.config" ` + -Force + + - task: PowerShell@2 + displayName: Filter signed packages and create Arcade V3 manifest + inputs: + targetType: inline + script: | + $ErrorActionPreference = 'Stop' + $outputRoot = '$(Build.ArtifactStagingDirectory)\SignedPackagePublishing' + New-Item -ItemType Directory -Force -Path ` + "$outputRoot\Audit", ` + "$outputRoot\PackageArtifacts", ` + "$outputRoot\AssetManifests" | Out-Null + + dotnet restore ` + '$(System.DefaultWorkingDirectory)\build\publishing\SignedPackagePublisher\SignedPackagePublisher.csproj' ` + --configfile '$(System.DefaultWorkingDirectory)\build\publishing\NuGet.config' ` + --nologo + if ($LastExitCode -ne 0) { + throw "Signed package publisher restore failed with exit code $LastExitCode." + } + + dotnet run ` + --project '$(System.DefaultWorkingDirectory)\build\publishing\SignedPackagePublisher\SignedPackagePublisher.csproj' ` + --configuration Release ` + --no-restore ` + -- ` + --packages '$(Pipeline.Workspace)\nuget-signed' ` + --package-output "$outputRoot\PackageArtifacts" ` + --inventory "$outputRoot\Audit\signed-package-inventory.json" ` + --manifest "$outputRoot\AssetManifests\Manifest.xml" ` + --feed '$(barTargetFeed)' ` + --max-concurrency '$(barFeedQueryConcurrency)' ` + --max-attempts '$(barFeedQueryAttempts)' ` + --repository-name '$(Build.Repository.Name)' ` + --build-number '$(Build.BuildNumber)' ` + --branch '$(Build.SourceBranch)' ` + --commit '$(Build.SourceVersion)' ` + --azure-collection-uri '$(System.CollectionUri)' ` + --azure-project '$(System.TeamProject)' ` + --azure-build-id '$(Build.BuildId)' ` + --azure-definition-id '$(System.DefinitionId)' ` + --azure-repository-uri '$(Build.Repository.Uri)' + if ($LastExitCode -ne 0) { + throw "Signed package filtering failed with exit code $LastExitCode." + } + + - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: true + args: + displayName: Publish signed package audit inventory + targetPath: $(Build.ArtifactStagingDirectory)\SignedPackagePublishing\Audit + artifactName: SignedPackagePublishingInventory + condition: always() + isProduction: false + + - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: true + args: + displayName: Publish filtered signed packages + targetPath: $(Build.ArtifactStagingDirectory)\SignedPackagePublishing\PackageArtifacts + artifactName: PackageArtifacts + condition: and(succeeded(), ne(variables['IncludedPackageCount'], '0')) + isProduction: true + + - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: true + args: + displayName: Publish filtered Arcade asset manifest + targetPath: $(Build.ArtifactStagingDirectory)\SignedPackagePublishing\AssetManifests + artifactName: AssetManifests + condition: and(succeeded(), ne(variables['IncludedPackageCount'], '0')) + isProduction: false + + - task: AzureCLI@2 + displayName: Register filtered signed packages in BAR + condition: and(succeeded(), ne(variables['IncludedPackageCount'], '0')) + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)\eng\common\sdk-task.ps1 + arguments: > + -task PublishBuildAssets + -restore + -msbuildEngine dotnet + /p:ManifestsPath='$(Build.ArtifactStagingDirectory)\SignedPackagePublishing\AssetManifests' + /p:IsAssetlessBuild=false + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:OfficialBuildId=$(Build.BuildNumber) + /p:RestoreConfigFile='$(System.DefaultWorkingDirectory)\build\publishing\NuGet.config' + env: + NUGET_CONFIG: $(System.DefaultWorkingDirectory)\build\publishing\NuGet.config + + - ${{ if ne(parameters.runValidation, true) }}: + - task: AzureCLI@2 + displayName: Publish BAR build through Darc default channels + condition: and(succeeded(), ne(variables['IncludedPackageCount'], '0')) + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)\eng\common\post-build\publish-using-darc.ps1 + arguments: > + -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(System.AccessToken)' + -WaitPublishingFinish true + -RequireDefaultChannels true + -SkipAssetsPublishing false + env: + NUGET_CONFIG: $(System.DefaultWorkingDirectory)\build\publishing\NuGet.config diff --git a/build/ci/variables.yml b/build/ci/variables.yml index d63f5d1fc..c7a946703 100644 --- a/build/ci/variables.yml +++ b/build/ci/variables.yml @@ -39,3 +39,10 @@ variables: # Signing settings TeamName: .NET MAUI + + # Signed package publishing remains disabled until the Azure DevOps environment + # and service connection setup in docs/publishing-signed-packages.md is complete. + barPublishingEnabled: 'false' + barTargetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json + barFeedQueryConcurrency: '8' + barFeedQueryAttempts: '4' diff --git a/build/publishing/NuGet.config b/build/publishing/NuGet.config new file mode 100644 index 000000000..d026c2322 --- /dev/null +++ b/build/publishing/NuGet.config @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/build/publishing/SignedPackagePublisher.Tests/FeedFailureClassifierTests.cs b/build/publishing/SignedPackagePublisher.Tests/FeedFailureClassifierTests.cs new file mode 100644 index 000000000..e3e6dcc80 --- /dev/null +++ b/build/publishing/SignedPackagePublisher.Tests/FeedFailureClassifierTests.cs @@ -0,0 +1,27 @@ +using System.Net; + +namespace SignedPackagePublisher.Tests; + +public sealed class FeedFailureClassifierTests +{ + [TestCase(HttpStatusCode.Unauthorized, FeedFailureKind.Authentication)] + [TestCase(HttpStatusCode.Forbidden, FeedFailureKind.Authentication)] + [TestCase(HttpStatusCode.RequestTimeout, FeedFailureKind.Transient)] + [TestCase(HttpStatusCode.TooManyRequests, FeedFailureKind.Transient)] + [TestCase(HttpStatusCode.InternalServerError, FeedFailureKind.Transient)] + [TestCase(HttpStatusCode.BadGateway, FeedFailureKind.Transient)] + [TestCase(HttpStatusCode.BadRequest, FeedFailureKind.Unknown)] + [TestCase(HttpStatusCode.NotFound, FeedFailureKind.Unknown)] + public void ClassifiesHttpFailuresStrictly(HttpStatusCode statusCode, FeedFailureKind expected) + => Assert.That(FeedFailureClassifier.Classify(statusCode), Is.EqualTo(expected)); + + [Test] + public void FindsStatusInInnerException() + { + var exception = new InvalidOperationException( + "wrapper", + new HttpRequestException("rate limited", null, HttpStatusCode.TooManyRequests)); + + Assert.That(FeedFailureClassifier.Classify(exception), Is.EqualTo(FeedFailureKind.Transient)); + } +} diff --git a/build/publishing/SignedPackagePublisher.Tests/GlobalUsings.cs b/build/publishing/SignedPackagePublisher.Tests/GlobalUsings.cs new file mode 100644 index 000000000..324456763 --- /dev/null +++ b/build/publishing/SignedPackagePublisher.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using NUnit.Framework; diff --git a/build/publishing/SignedPackagePublisher.Tests/PackageInspectorTests.cs b/build/publishing/SignedPackagePublisher.Tests/PackageInspectorTests.cs new file mode 100644 index 000000000..e530af428 --- /dev/null +++ b/build/publishing/SignedPackagePublisher.Tests/PackageInspectorTests.cs @@ -0,0 +1,60 @@ +using System.IO.Compression; + +namespace SignedPackagePublisher.Tests; + +public sealed class PackageInspectorTests +{ + [Test] + public async Task ReadsIdentityAndNormalizesVersionFromNuspec() + { + using var directory = new TemporaryDirectory(); + CreatePackage(Path.Combine(directory.Path, "not-the-identity.nupkg"), "Example.Package", "1.2.3.0"); + + IReadOnlyList packages = await PackageInspector.InspectDirectoryAsync( + directory.Path, + CancellationToken.None); + + Assert.Multiple(() => { + Assert.That(packages, Has.Count.EqualTo(1)); + Assert.That(packages[0].Id, Is.EqualTo("Example.Package")); + Assert.That(packages[0].Version.ToNormalizedString(), Is.EqualTo("1.2.3")); + Assert.That(packages[0].Sha256, Has.Length.EqualTo(64)); + }); + } + + internal static void CreatePackage(string path, string id, string version, string content = "content") + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + using ZipArchive archive = ZipFile.Open(path, ZipArchiveMode.Create); + ZipArchiveEntry nuspec = archive.CreateEntry($"{id}.nuspec"); + using (var writer = new StreamWriter(nuspec.Open())) + { + writer.Write($""" + + + + {id} + {version} + Test + Test + + + """); + } + using var contentWriter = new StreamWriter(archive.CreateEntry("content.txt").Open()); + contentWriter.Write(content); + } +} + +internal sealed class TemporaryDirectory : IDisposable +{ + public TemporaryDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"signed-package-publisher-{Guid.NewGuid():N}"); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() => Directory.Delete(Path, recursive: true); +} diff --git a/build/publishing/SignedPackagePublisher.Tests/PublishingOutputTests.cs b/build/publishing/SignedPackagePublisher.Tests/PublishingOutputTests.cs new file mode 100644 index 000000000..d43d43381 --- /dev/null +++ b/build/publishing/SignedPackagePublisher.Tests/PublishingOutputTests.cs @@ -0,0 +1,63 @@ +using System.Text.Json; + +namespace SignedPackagePublisher.Tests; + +public sealed class PublishingOutputTests +{ + [Test] + public async Task InventoryIsByteForByteDeterministic() + { + using var directory = new TemporaryDirectory(); + var plan = new PublishingPlan( + new[] { + new PackageInventoryEntry("a.nupkg", "a.nupkg", "aa", "A.Package", "1.0.0", PackageDecision.Include, "missing-from-feed"), + }, + Array.Empty()); + string first = Path.Combine(directory.Path, "first.json"); + string second = Path.Combine(directory.Path, "second.json"); + var feed = new Uri("https://example.test/v3/index.json"); + + await PublishingOutput.WriteInventoryAsync(first, feed, plan, CancellationToken.None); + await PublishingOutput.WriteInventoryAsync(second, feed, plan, CancellationToken.None); + + Assert.That(await File.ReadAllBytesAsync(first), Is.EqualTo(await File.ReadAllBytesAsync(second))); + using JsonDocument document = JsonDocument.Parse(await File.ReadAllTextAsync(first)); + Assert.That(document.RootElement.GetProperty("packages").GetArrayLength(), Is.EqualTo(1)); + } + + [Test] + public void ManifestMarksAssetsAsShippingPackageCategory() + { + using var directory = new TemporaryDirectory(); + string manifestPath = Path.Combine(directory.Path, "Manifest.xml"); + var package = new SignedPackage( + "a.nupkg", + "a.nupkg", + "a.nupkg", + "aa", + "A.Package", + NuGet.Versioning.NuGetVersion.Parse("1.0.0")); + + PublishingOutput.WriteManifest( + manifestPath, + new ManifestIdentity( + "dotnet/android-libraries", + "20260817.1", + "refs/heads/main", + new string('a', 40), + "https://dev.azure.com/devdiv/", + "DevDiv", + 123, + 456, + "https://dev.azure.com/devdiv/DevDiv/_git/android-libraries"), + new[] { package }); + + string xml = File.ReadAllText(manifestPath); + Assert.Multiple(() => { + Assert.That(xml, Does.Contain("PublishingVersion=\"3\"")); + Assert.That(xml, Does.Contain("Id=\"A.Package\"")); + Assert.That(xml, Does.Contain("NonShipping=\"False\"")); + Assert.That(xml, Does.Contain("Category=\"Package\"")); + }); + } +} diff --git a/build/publishing/SignedPackagePublisher.Tests/PublishingPlannerTests.cs b/build/publishing/SignedPackagePublisher.Tests/PublishingPlannerTests.cs new file mode 100644 index 000000000..03ea88a33 --- /dev/null +++ b/build/publishing/SignedPackagePublisher.Tests/PublishingPlannerTests.cs @@ -0,0 +1,134 @@ +using NuGet.Versioning; + +namespace SignedPackagePublisher.Tests; + +public sealed class PublishingPlannerTests +{ + [Test] + public async Task FiltersExistingPackagesAndProducesDeterministicInventory() + { + var packages = new[] { + Package("z.nupkg", "Z.Package", "2.0.0", "bb"), + Package("a.nupkg", "A.Package", "1.0.0", "aa"), + }; + var probe = new StubProbe(("A.Package", "1.0.0", true), ("Z.Package", "2.0.0", false)); + + PublishingPlan plan = await new PublishingPlanner(probe, 2).CreateAsync(packages, CancellationToken.None); + + Assert.Multiple(() => { + Assert.That(plan.Inventory.Select(entry => entry.Id), Is.EqualTo(new[] { "A.Package", "Z.Package" })); + Assert.That(plan.Inventory.Select(entry => entry.Reason), Is.EqualTo(new[] { "already-exists", "missing-from-feed" })); + Assert.That(plan.IncludedPackages.Select(package => package.Id), Is.EqualTo(new[] { "Z.Package" })); + }); + } + + [Test] + public async Task DeduplicatesIdenticalIdentityAndContent() + { + var packages = new[] { + Package("b/copy.nupkg", "Example.Package", "1.0.0", "aa"), + Package("a/original.nupkg", "Example.Package", "1.0.0", "aa"), + }; + var probe = new StubProbe(("Example.Package", "1.0.0", false)); + + PublishingPlan plan = await new PublishingPlanner(probe, 1).CreateAsync(packages, CancellationToken.None); + + Assert.Multiple(() => { + Assert.That(plan.HasErrors, Is.False); + Assert.That(plan.IncludedPackages.Single().RelativePath, Is.EqualTo("a/original.nupkg")); + Assert.That(plan.Inventory.Single(entry => entry.SourceFile == "b/copy.nupkg").Reason, Is.EqualTo("duplicate-identical")); + }); + } + + [Test] + public async Task RejectsDuplicateIdentityWithDifferentContent() + { + var packages = new[] { + Package("a.nupkg", "Example.Package", "1.0.0", "aa"), + Package("b.nupkg", "Example.Package", "1.0.0", "bb"), + }; + + PublishingPlan plan = await new PublishingPlanner(new StubProbe(), 1) + .CreateAsync(packages, CancellationToken.None); + + Assert.Multiple(() => { + Assert.That(plan.HasErrors, Is.True); + Assert.That(plan.Inventory, Has.All.Property(nameof(PackageInventoryEntry.Reason)).EqualTo("duplicate-identity-different-content")); + }); + } + + [Test] + public async Task RejectsDuplicateFilenameWithDifferentIdentities() + { + var packages = new[] { + Package("a/same.nupkg", "First.Package", "1.0.0", "aa"), + Package("b/same.nupkg", "Second.Package", "1.0.0", "bb"), + }; + + PublishingPlan plan = await new PublishingPlanner(new StubProbe(), 1) + .CreateAsync(packages, CancellationToken.None); + + Assert.That(plan.Inventory, Has.All.Property(nameof(PackageInventoryEntry.Reason)).EqualTo("duplicate-filename-different-identity")); + } + + [Test] + public async Task AuditsIdentityDuplicatesRelatedToFilenameCollision() + { + var packages = new[] { + Package("a/same.nupkg", "First.Package", "1.0.0", "aa"), + Package("b/same.nupkg", "Second.Package", "1.0.0", "bb"), + Package("c/other.nupkg", "First.Package", "1.0.0", "aa"), + }; + + PublishingPlan plan = await new PublishingPlanner(new StubProbe(), 1) + .CreateAsync(packages, CancellationToken.None); + + Assert.Multiple(() => { + Assert.That(plan.Inventory, Has.Count.EqualTo(3)); + Assert.That( + plan.Inventory.Single(entry => entry.SourceFile == "c/other.nupkg").Reason, + Is.EqualTo("duplicate-identity-in-conflicting-filename-set")); + }); + } + + [Test] + public async Task TreatsFeedFailuresAsErrorsNotExistingPackages() + { + var package = Package("a.nupkg", "Example.Package", "1.0.0", "aa"); + var probe = new StubProbe(new FeedQueryException("auth failed", new UnauthorizedAccessException())); + + PublishingPlan plan = await new PublishingPlanner(probe, 1) + .CreateAsync(new[] { package }, CancellationToken.None); + + Assert.Multiple(() => { + Assert.That(plan.HasErrors, Is.True); + Assert.That(plan.Inventory.Single().Decision, Is.EqualTo(PackageDecision.Error)); + Assert.That(plan.Inventory.Single().Reason, Does.StartWith("feed-query-failed:")); + }); + } + + private static SignedPackage Package(string relativePath, string id, string version, string hash) + => new(relativePath, relativePath, Path.GetFileName(relativePath), hash, id, NuGetVersion.Parse(version)); + + private sealed class StubProbe : IPackageFeedProbe + { + private readonly Dictionary<(string Id, string Version), bool> results; + private readonly Exception? exception; + + public StubProbe(params (string Id, string Version, bool Exists)[] results) + => this.results = results.ToDictionary( + result => (result.Id, result.Version), + result => result.Exists); + + public StubProbe(Exception exception) + { + results = []; + this.exception = exception; + } + + public Task ExistsAsync(string id, NuGetVersion version, CancellationToken cancellationToken) + => exception is null + ? Task.FromResult(results[(id, version.ToNormalizedString())]) + : Task.FromException(exception); + } +} diff --git a/build/publishing/SignedPackagePublisher.Tests/SignedPackagePublisher.Tests.csproj b/build/publishing/SignedPackagePublisher.Tests/SignedPackagePublisher.Tests.csproj new file mode 100644 index 000000000..b43726952 --- /dev/null +++ b/build/publishing/SignedPackagePublisher.Tests/SignedPackagePublisher.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + true + false + true + + + + + + + + + diff --git a/build/publishing/SignedPackagePublisher/Models.cs b/build/publishing/SignedPackagePublisher/Models.cs new file mode 100644 index 000000000..93287d934 --- /dev/null +++ b/build/publishing/SignedPackagePublisher/Models.cs @@ -0,0 +1,45 @@ +using NuGet.Versioning; + +namespace SignedPackagePublisher; + +public enum PackageDecision +{ + Include, + Exclude, + Error, +} + +public sealed record SignedPackage( + string SourcePath, + string RelativePath, + string FileName, + string Sha256, + string Id, + NuGetVersion Version); + +public sealed record PackageInventoryEntry( + string SourceFile, + string FileName, + string Sha256, + string Id, + string Version, + PackageDecision Decision, + string Reason, + string? CanonicalSourceFile = null); + +public sealed record PublishingPlan( + IReadOnlyList Inventory, + IReadOnlyList IncludedPackages) +{ + public bool HasErrors => Inventory.Any(entry => entry.Decision == PackageDecision.Error); +} + +public enum FeedFailureKind +{ + Transient, + Authentication, + Unknown, +} + +public sealed class FeedQueryException(string message, Exception innerException) + : Exception(message, innerException); diff --git a/build/publishing/SignedPackagePublisher/NuGetFeedProbe.cs b/build/publishing/SignedPackagePublisher/NuGetFeedProbe.cs new file mode 100644 index 000000000..3416679e1 --- /dev/null +++ b/build/publishing/SignedPackagePublisher/NuGetFeedProbe.cs @@ -0,0 +1,107 @@ +using System.Net; +using NuGet.Common; +using NuGet.Configuration; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; + +namespace SignedPackagePublisher; + +public interface IPackageFeedProbe +{ + Task ExistsAsync(string id, NuGet.Versioning.NuGetVersion version, CancellationToken cancellationToken); +} + +public sealed class NuGetFeedProbe : IPackageFeedProbe, IDisposable +{ + private readonly SourceRepository repository; + private readonly SourceCacheContext cacheContext = new() { + NoCache = true, + DirectDownload = true, + }; + private readonly int maxAttempts; + + public NuGetFeedProbe(Uri serviceIndex, string? token, int maxAttempts) + { + if (!serviceIndex.IsAbsoluteUri) + throw new ArgumentException("The NuGet service index must be an absolute URI.", nameof(serviceIndex)); + if (maxAttempts < 1) + throw new ArgumentOutOfRangeException(nameof(maxAttempts)); + + var packageSource = new PackageSource(serviceIndex.AbsoluteUri); + if (!string.IsNullOrWhiteSpace(token)) + { + packageSource.Credentials = PackageSourceCredential.FromUserInput( + serviceIndex.AbsoluteUri, + "AzureDevOps", + token, + storePasswordInClearText: true, + validAuthenticationTypesText: "basic"); + } + + repository = Repository.Factory.GetCoreV3(packageSource); + this.maxAttempts = maxAttempts; + } + + public async Task ExistsAsync( + string id, + NuGet.Versioning.NuGetVersion version, + CancellationToken cancellationToken) + { + for (int attempt = 1; ; attempt++) + { + try + { + FindPackageByIdResource resource = await repository.GetResourceAsync(cancellationToken); + return await resource.DoesPackageExistAsync(id, version, cacheContext, NullLogger.Instance, cancellationToken); + } + catch (Exception exception) when (attempt < maxAttempts + && FeedFailureClassifier.Classify(exception) == FeedFailureKind.Transient) + { + await Task.Delay(Backoff(attempt), cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException + || !cancellationToken.IsCancellationRequested) + { + FeedFailureKind kind = FeedFailureClassifier.Classify(exception); + throw new FeedQueryException( + $"NuGet feed lookup failed for '{id} {version.ToNormalizedString()}' ({kind}).", + exception); + } + } + } + + public void Dispose() => cacheContext.Dispose(); + + private static TimeSpan Backoff(int attempt) + => TimeSpan.FromMilliseconds(Math.Min(5_000, 250 * Math.Pow(2, attempt - 1))); +} + +public static class FeedFailureClassifier +{ + public static FeedFailureKind Classify(Exception exception) + { + foreach (Exception current in Enumerate(exception)) + { + if (current is HttpRequestException httpException && httpException.StatusCode is HttpStatusCode statusCode) + return Classify(statusCode); + + if (current is TaskCanceledException) + return FeedFailureKind.Transient; + } + + return FeedFailureKind.Unknown; + } + + public static FeedFailureKind Classify(HttpStatusCode statusCode) => statusCode switch { + HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden => FeedFailureKind.Authentication, + HttpStatusCode.RequestTimeout or HttpStatusCode.TooManyRequests => FeedFailureKind.Transient, + >= HttpStatusCode.InternalServerError => FeedFailureKind.Transient, + _ => FeedFailureKind.Unknown, + }; + + private static IEnumerable Enumerate(Exception exception) + { + for (Exception? current = exception; current is not null; current = current.InnerException) + yield return current; + } +} diff --git a/build/publishing/SignedPackagePublisher/PackageInspector.cs b/build/publishing/SignedPackagePublisher/PackageInspector.cs new file mode 100644 index 000000000..f7cee777b --- /dev/null +++ b/build/publishing/SignedPackagePublisher/PackageInspector.cs @@ -0,0 +1,48 @@ +using System.Security.Cryptography; +using NuGet.Packaging; + +namespace SignedPackagePublisher; + +public static class PackageInspector +{ + public static async Task> InspectDirectoryAsync( + string packagesDirectory, + CancellationToken cancellationToken) + { + string root = Path.GetFullPath(packagesDirectory); + if (!Directory.Exists(root)) + throw new DirectoryNotFoundException($"Signed package directory '{root}' does not exist."); + + string[] packagePaths = Directory.GetFiles(root, "*.nupkg", SearchOption.AllDirectories); + if (packagePaths.Length == 0) + throw new InvalidOperationException($"No signed .nupkg files were found in '{root}'."); + + var packages = new List(packagePaths.Length); + foreach (string packagePath in packagePaths.Order(StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + string hash = await ComputeSha256Async(packagePath, cancellationToken); + + using var reader = new PackageArchiveReader(packagePath); + var identity = await reader.GetIdentityAsync(cancellationToken) + ?? throw new InvalidDataException($"Package '{packagePath}' has no NuGet identity."); + + packages.Add(new SignedPackage( + packagePath, + Path.GetRelativePath(root, packagePath).Replace('\\', '/'), + Path.GetFileName(packagePath), + hash, + identity.Id, + identity.Version)); + } + + return packages; + } + + private static async Task ComputeSha256Async(string path, CancellationToken cancellationToken) + { + await using FileStream stream = File.OpenRead(path); + byte[] hash = await SHA256.HashDataAsync(stream, cancellationToken); + return Convert.ToHexStringLower(hash); + } +} diff --git a/build/publishing/SignedPackagePublisher/Program.cs b/build/publishing/SignedPackagePublisher/Program.cs new file mode 100644 index 000000000..6b5cae3f2 --- /dev/null +++ b/build/publishing/SignedPackagePublisher/Program.cs @@ -0,0 +1,103 @@ +namespace SignedPackagePublisher; + +public static class Program +{ + public static async Task Main(string[] args) + { + try + { + Options options = Options.Parse(args); + using var probe = new NuGetFeedProbe( + options.Feed, + GetOptionalEnvironmentVariable(options.FeedTokenEnvironmentVariable), + options.MaxAttempts); + + IReadOnlyList packages = await PackageInspector.InspectDirectoryAsync( + options.PackagesDirectory, + CancellationToken.None); + var planner = new PublishingPlanner(probe, options.MaxConcurrency); + PublishingPlan plan = await planner.CreateAsync(packages, CancellationToken.None); + await PublishingOutput.WriteInventoryAsync( + options.InventoryPath, + options.Feed, + plan, + CancellationToken.None); + + if (plan.HasErrors) + { + Console.Error.WriteLine("Package filtering failed. See the deterministic audit inventory for details."); + return 2; + } + + PublishingOutput.StageIncludedPackages(options.PackageOutputDirectory, plan.IncludedPackages); + PublishingOutput.WriteManifest(options.ManifestPath, options.ManifestIdentity, plan.IncludedPackages); + Console.WriteLine($"Inspected {packages.Count} signed packages; {plan.IncludedPackages.Count} are absent from the target feed."); + Console.WriteLine($"##vso[task.setvariable variable=IncludedPackageCount]{plan.IncludedPackages.Count}"); + return 0; + } + catch (Exception exception) + { + Console.Error.WriteLine(exception); + return 1; + } + } + + private static string? GetOptionalEnvironmentVariable(string? name) + => string.IsNullOrWhiteSpace(name) ? null : Environment.GetEnvironmentVariable(name); +} + +public sealed record Options( + string PackagesDirectory, + string PackageOutputDirectory, + string InventoryPath, + string ManifestPath, + Uri Feed, + string? FeedTokenEnvironmentVariable, + int MaxConcurrency, + int MaxAttempts, + ManifestIdentity ManifestIdentity) +{ + public static Options Parse(string[] args) + { + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int index = 0; index < args.Length; index += 2) + { + if (index + 1 >= args.Length || !args[index].StartsWith("--", StringComparison.Ordinal)) + throw new ArgumentException("Arguments must be supplied as '--name value' pairs."); + values.Add(args[index][2..], args[index + 1]); + } + + string Required(string name) + => values.TryGetValue(name, out string? value) && !string.IsNullOrWhiteSpace(value) + ? value + : throw new ArgumentException($"Missing required argument '--{name}'."); + int RequiredInt(string name) + => int.TryParse(Required(name), out int value) + ? value + : throw new ArgumentException($"Argument '--{name}' must be an integer."); + int OptionalInt(string name, int fallback) + => values.TryGetValue(name, out string? value) + ? int.Parse(value, System.Globalization.CultureInfo.InvariantCulture) + : fallback; + + return new Options( + Required("packages"), + Required("package-output"), + Required("inventory"), + Required("manifest"), + new Uri(Required("feed"), UriKind.Absolute), + values.GetValueOrDefault("feed-token-env"), + OptionalInt("max-concurrency", 8), + OptionalInt("max-attempts", 4), + new ManifestIdentity( + Required("repository-name"), + Required("build-number"), + Required("branch"), + Required("commit"), + Required("azure-collection-uri"), + Required("azure-project"), + RequiredInt("azure-build-id"), + RequiredInt("azure-definition-id"), + Required("azure-repository-uri"))); + } +} diff --git a/build/publishing/SignedPackagePublisher/PublishingOutput.cs b/build/publishing/SignedPackagePublisher/PublishingOutput.cs new file mode 100644 index 000000000..db29d3cea --- /dev/null +++ b/build/publishing/SignedPackagePublisher/PublishingOutput.cs @@ -0,0 +1,115 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Xml.Linq; +using Microsoft.DotNet.Build.Manifest; + +namespace SignedPackagePublisher; + +public sealed record ManifestIdentity( + string RepositoryName, + string BuildNumber, + string Branch, + string Commit, + string AzureCollectionUri, + string AzureProject, + int AzureBuildId, + int AzureDefinitionId, + string AzureRepositoryUri); + +public sealed record InventoryDocument( + int SchemaVersion, + string Feed, + IReadOnlyList Packages); + +[JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + UseStringEnumConverter = true)] +[JsonSerializable(typeof(InventoryDocument))] +internal sealed partial class InventoryJsonContext : JsonSerializerContext; + +public static class PublishingOutput +{ + public static async Task WriteInventoryAsync( + string path, + Uri feed, + PublishingPlan plan, + CancellationToken cancellationToken) + { + EnsureParentDirectory(path); + var document = new InventoryDocument(1, feed.AbsoluteUri, plan.Inventory); + await using FileStream stream = File.Create(path); + await JsonSerializer.SerializeAsync( + stream, + document, + InventoryJsonContext.Default.InventoryDocument, + cancellationToken); + await stream.WriteAsync("\n"u8.ToArray(), cancellationToken); + } + + public static void StageIncludedPackages(string outputDirectory, IReadOnlyList packages) + { + Directory.CreateDirectory(outputDirectory); + foreach (SignedPackage package in packages) + { + string destination = Path.Combine( + outputDirectory, + $"{package.Id}.{package.Version.ToNormalizedString()}.nupkg"); + File.Copy(package.SourcePath, destination, overwrite: false); + } + } + + public static void WriteManifest( + string path, + ManifestIdentity identity, + IReadOnlyList packages) + { + var build = new BuildModel(new BuildIdentity { + PublishingVersion = PublishingInfraVersion.V3, + Name = identity.RepositoryName, + BuildId = identity.BuildNumber, + Branch = identity.Branch, + Commit = identity.Commit, + IsStable = false, + IsReleaseOnlyPackageVersion = false, + InitialAssetsLocation = $"{identity.AzureCollectionUri.TrimEnd('/')}/{identity.AzureProject}/_apis/build/builds/{identity.AzureBuildId}/artifacts", + AzureDevOpsAccount = GetAzureDevOpsAccount(identity.AzureCollectionUri), + AzureDevOpsProject = identity.AzureProject, + AzureDevOpsBuildNumber = identity.BuildNumber, + AzureDevOpsRepository = identity.AzureRepositoryUri, + AzureDevOpsBranch = identity.Branch, + AzureDevOpsBuildId = identity.AzureBuildId, + AzureDevOpsBuildDefinitionId = identity.AzureDefinitionId, + }); + + foreach (SignedPackage package in packages) + { + var asset = new PackageArtifactModel { + Id = package.Id, + Version = package.Version.ToNormalizedString(), + NonShipping = false, + }; + asset.Attributes["Category"] = "Package"; + build.Artifacts.Packages.Add(asset); + } + + EnsureParentDirectory(path); + File.WriteAllText(path, build.ToXml().ToString(SaveOptions.DisableFormatting)); + } + + private static void EnsureParentDirectory(string path) + { + string? directory = Path.GetDirectoryName(Path.GetFullPath(path)); + if (directory is not null) + Directory.CreateDirectory(directory); + } + + private static string GetAzureDevOpsAccount(string collectionUri) + { + var uri = new Uri(collectionUri); + if (uri.Host.Equals("dev.azure.com", StringComparison.OrdinalIgnoreCase)) + return uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries).First(); + + return uri.Host.Split('.', StringSplitOptions.RemoveEmptyEntries).First(); + } +} diff --git a/build/publishing/SignedPackagePublisher/PublishingPlanner.cs b/build/publishing/SignedPackagePublisher/PublishingPlanner.cs new file mode 100644 index 000000000..6b06e94d0 --- /dev/null +++ b/build/publishing/SignedPackagePublisher/PublishingPlanner.cs @@ -0,0 +1,123 @@ +using NuGet.Packaging.Core; +using System.Collections.Concurrent; + +namespace SignedPackagePublisher; + +public sealed class PublishingPlanner(IPackageFeedProbe feedProbe, int maxConcurrency) +{ + private static readonly PackageIdentityComparer IdentityComparer = PackageIdentityComparer.Default; + + public async Task CreateAsync( + IReadOnlyList packages, + CancellationToken cancellationToken) + { + if (maxConcurrency < 1) + throw new ArgumentOutOfRangeException(nameof(maxConcurrency)); + + var entries = new ConcurrentDictionary(StringComparer.Ordinal); + var canonicalPackages = new List(); + + foreach (IGrouping fileNameGroup in packages.GroupBy( + package => package.FileName, + StringComparer.OrdinalIgnoreCase)) + { + var identities = fileNameGroup + .Select(package => new PackageIdentity(package.Id, package.Version)) + .Distinct(IdentityComparer) + .ToArray(); + if (identities.Length > 1) + { + foreach (SignedPackage package in fileNameGroup) + entries[package.RelativePath] = Entry(package, PackageDecision.Error, "duplicate-filename-different-identity"); + } + } + + foreach (IGrouping identityGroup in packages.GroupBy( + package => new PackageIdentity(package.Id, package.Version), + IdentityComparer)) + { + SignedPackage[] ordered = identityGroup.OrderBy(package => package.RelativePath, StringComparer.Ordinal).ToArray(); + SignedPackage[] unrecorded = ordered.Where(package => !entries.ContainsKey(package.RelativePath)).ToArray(); + if (unrecorded.Length == 0) + continue; + if (unrecorded.Length != ordered.Length) + { + foreach (SignedPackage package in unrecorded) + entries[package.RelativePath] = Entry(package, PackageDecision.Error, "duplicate-identity-in-conflicting-filename-set"); + continue; + } + + if (ordered.Select(package => package.Sha256).Distinct(StringComparer.Ordinal).Count() > 1) + { + foreach (SignedPackage package in ordered) + entries[package.RelativePath] = Entry(package, PackageDecision.Error, "duplicate-identity-different-content"); + continue; + } + + SignedPackage canonical = ordered[0]; + canonicalPackages.Add(canonical); + foreach (SignedPackage duplicate in ordered.Skip(1)) + { + entries[duplicate.RelativePath] = Entry( + duplicate, + PackageDecision.Exclude, + "duplicate-identical", + canonical.RelativePath); + } + } + + using var gate = new SemaphoreSlim(maxConcurrency, maxConcurrency); + await Task.WhenAll(canonicalPackages.Select(async package => { + await gate.WaitAsync(cancellationToken); + try + { + bool exists = await feedProbe.ExistsAsync(package.Id, package.Version, cancellationToken); + entries[package.RelativePath] = Entry( + package, + exists ? PackageDecision.Exclude : PackageDecision.Include, + exists ? "already-exists" : "missing-from-feed"); + } + catch (Exception exception) when (exception is not OperationCanceledException + || !cancellationToken.IsCancellationRequested) + { + entries[package.RelativePath] = Entry( + package, + PackageDecision.Error, + $"feed-query-failed: {exception.Message}"); + } + finally + { + gate.Release(); + } + })); + + PackageInventoryEntry[] inventory = entries.Values + .OrderBy(entry => entry.Id, StringComparer.OrdinalIgnoreCase) + .ThenBy(entry => entry.Version, StringComparer.OrdinalIgnoreCase) + .ThenBy(entry => entry.SourceFile, StringComparer.Ordinal) + .ToArray(); + + SignedPackage[] included = canonicalPackages + .Where(package => entries[package.RelativePath].Decision == PackageDecision.Include) + .OrderBy(package => package.Id, StringComparer.OrdinalIgnoreCase) + .ThenBy(package => package.Version) + .ToArray(); + + return new PublishingPlan(inventory, included); + } + + private static PackageInventoryEntry Entry( + SignedPackage package, + PackageDecision decision, + string reason, + string? canonicalSourceFile = null) + => new( + package.RelativePath, + package.FileName, + package.Sha256, + package.Id, + package.Version.ToNormalizedString(), + decision, + reason, + canonicalSourceFile); +} diff --git a/build/publishing/SignedPackagePublisher/SignedPackagePublisher.csproj b/build/publishing/SignedPackagePublisher/SignedPackagePublisher.csproj new file mode 100644 index 000000000..8753e6f0e --- /dev/null +++ b/build/publishing/SignedPackagePublisher/SignedPackagePublisher.csproj @@ -0,0 +1,20 @@ + + + Exe + net10.0 + enable + enable + true + false + false + + 11.0.0-beta.26381.103 + + + + + + + + + diff --git a/docs/publishing-signed-packages.md b/docs/publishing-signed-packages.md new file mode 100644 index 000000000..1baded1d8 --- /dev/null +++ b/docs/publishing-signed-packages.md @@ -0,0 +1,24 @@ +# Publishing signed packages through BAR + +The `publish_signed_packages` stage consumes only the `nuget-signed` pipeline artifact produced by the existing Xamarin signing job. It reads each signed `.nupkg` identity and normalized version, queries that exact package in the public `dotnet10` NuGet V3 feed, and creates an Arcade V3 manifest containing only missing packages. Every inspected package and its SHA-256 is recorded in the `SignedPackagePublishingInventory` pipeline artifact. + +The manifest uses Arcade's `PackageArtifactModel` with `Category=Package` and `NonShipping=false`. Arcade's .NET 10 public channel maps that combination to the `dotnet10` shipping feed. The stage does not consume or register `output-windows`, and it does not publish to NuGet.org. + +## One-time Azure DevOps setup + +Publishing is intentionally disabled by `barPublishingEnabled: false` until all setup below is complete: + +1. Authorize the AndroidX pipeline to use the `Darc: Maestro Production` service connection. +2. Add an **Exclusive lock** check to that service connection. The stage sets `lockBehavior: sequential`, so the feed check, BAR registration, and promotion remain in one serialized critical section. +3. Configure the repository's `main` default channel in Darc to the intended public .NET 10 channel, and confirm that channel maps shipping `Package` assets to `https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json`. Publishing uses Arcade's vendored `publish-using-darc.ps1` with `--default-channels-required`, so it fails safely when no default channel is configured. +4. Set `barPublishingEnabled` to `true` in `build/ci/variables.yml`. + +The public `dotnet10` feed currently permits anonymous reads. If that policy changes, supply a read token through a secret environment variable and pass its name with `--feed-token-env`; the tool never accepts a token value on its command line. + +The stage uses the same real-sign condition as `build/ci/stage-sign-artifacts.yml`: non-PR `release/*` builds and non-scheduled `main` builds. PRs, scheduled builds, public validation, and all test-signed artifacts are excluded. + +Publishing tools restore through `build/publishing/NuGet.config`, which adds `dotnet-eng` only for this isolated publishing surface. The repository-wide `NuGet.config` and normal Cake restore sources remain unchanged. + +## Validate BAR registration without promotion + +For a manual feature-branch run, set `RunBarValidation=true` and `BarValidationSignedBuildId` to a successful non-scheduled `main` or `release/*` AndroidX build containing `nuget-signed`. Validation mode rejects test-signed source builds, registers the filtered real-signed assets in BAR, and omits the channel-promotion task from the compiled job. The validation build therefore remains unassociated with `.NET 10` and cannot publish packages to the `dotnet10` feed. diff --git a/global.json b/global.json index d1e00d327..a552e93f2 100644 --- a/global.json +++ b/global.json @@ -1,4 +1,7 @@ { + "tools": { + "dotnet": "11.0.100-preview.7.26381.103" + }, "msbuild-sdks": { "Microsoft.Build.Traversal": "4.1.0", "Microsoft.Build.NoTargets": "3.7.56",