diff --git a/docs/detectors/swift.md b/docs/detectors/swift.md index 3b44ef630..59aad5f74 100644 --- a/docs/detectors/swift.md +++ b/docs/detectors/swift.md @@ -14,6 +14,17 @@ The version, the url and commit hash of the dependencies are stored in this file [This is the only reference in the Apple documentation to the `Package.resolved` file.][1] +## Experiment results + +Swift detection is experimental. The registered `SwiftDetectorExperiment` uses an empty control group +and `SwiftResolvedComponentDetector` as its experiment group, so all Swift discoveries are reported as additions. +It does not depend on CocoaPods detection or compare Swift packages against CocoaPods packages. + +Enable experiment processing with `CD_DETECTOR_EXPERIMENTS` or `DetectorExperiments.Enable`. +Scans with no Swift components do not produce experiment results. Disabling the Swift detector also disables this experiment. +The CLI writes results to `Experiment_SwiftDetectorExperiment_{timestamp}_{processId}.json`; +hosts that emit `ExperimentResults` custom events must register their own `IExperimentProcessor`. + ## Known limitations Right now the detector does not support parsing `Package.swift` file to get the dependencies. diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/SwiftDetectorExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/SwiftDetectorExperiment.cs new file mode 100644 index 000000000..dc25a409a --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/SwiftDetectorExperiment.cs @@ -0,0 +1,22 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.Swift; + +/// +/// Records Swift Package Manager discoveries against an empty baseline. +/// +public class SwiftDetectorExperiment : IExperimentConfiguration +{ + /// + public string Name => "SwiftDetectorExperiment"; + + /// + public bool IsInControlGroup(IComponentDetector componentDetector) => false; + + /// + public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is SwiftResolvedComponentDetector; + + /// + public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true; +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index dd33c8a83..edd1f0031 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -74,6 +74,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // Detectors // CocoaPods diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentConfigTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentConfigTests.cs index 433362734..7519ca637 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentConfigTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentConfigTests.cs @@ -35,6 +35,12 @@ public void ExperimentConfig_HaveDistinctNames() names.Should().OnlyHaveUniqueItems(); } + [TestMethod] + public void ExperimentConfig_RegistersSwiftDetectorExperiment() + { + this.configs.OfType().Should().ContainSingle(); + } + [TestMethod] public async Task ExperimentConfig_InitShouldNotThrowAsync() { diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index ee369f242..4d8173f57 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -4,12 +4,16 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Experiments; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Common.DependencyGraph; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.CocoaPods; using Microsoft.ComponentDetection.Detectors.NuGet; +using Microsoft.ComponentDetection.Detectors.Swift; using Microsoft.ComponentDetection.Orchestrator.Commands; using Microsoft.ComponentDetection.Orchestrator.Experiments; using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; @@ -165,6 +169,69 @@ public async Task FinishAsync_ProcessesExperimentsAsync() Times.Once()); } + [TestMethod] + public async Task FinishAsync_ProcessesSwiftExperimentWithoutControlDetectorAsync() + { + var config = new SwiftDetectorExperiment(); + var swiftDetector = new SwiftResolvedComponentDetector(null, null, null); + var podDetector = new PodComponentDetector(null, null, null); + var service = new ExperimentService( + [config], + [this.experimentProcessorMock.Object], + this.graphTranslationServiceMock.Object, + this.loggerMock.Object); + service.RemoveUnwantedExperimentsbyDetectors([podDetector]); + + this.SetupGraphMock([]); + service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.scanSettingsMock.Object); + + var component = new SwiftComponent + { + Name = "alamofire", + Version = "5.9.1", + RepositoryUrl = new Uri("https://github.com/Alamofire/Alamofire"), + CommitHash = "f455c2975872ccd2d9c81594c658af65716e9b9a", + Kind = "remoteSourceControl", + }; + this.SetupGraphMock([new ScannedComponent { Component = component }]); + service.RecordDetectorRun(swiftDetector, this.componentRecorder, this.scanSettingsMock.Object); + + await service.FinishAsync(); + + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync( + config, + It.Is(diff => + diff.AddedIds.Count == 1 && diff.AddedIds.Contains(component.Id) + && diff.RemovedIds.Count == 0 + && diff.ControlDetectors.Count == 0 + && diff.ExperimentalDetectors.Count == 1 + && diff.ExperimentalDetectors.Single().DetectorId == swiftDetector.Id)), + Times.Once()); + } + + [TestMethod] + public async Task FinishAsync_SkipsSwiftExperimentWithoutSwiftComponentsAsync() + { + var config = new SwiftDetectorExperiment(); + var service = new ExperimentService( + [config], + [this.experimentProcessorMock.Object], + this.graphTranslationServiceMock.Object, + this.loggerMock.Object); + this.SetupGraphMock(ExperimentTestUtils.CreateRandomComponents()); + service.RecordDetectorRun(new PodComponentDetector(null, null, null), this.componentRecorder, this.scanSettingsMock.Object); + + this.SetupGraphMock([]); + service.RecordDetectorRun(new SwiftResolvedComponentDetector(null, null, null), this.componentRecorder, this.scanSettingsMock.Object); + + await service.FinishAsync(); + + this.experimentProcessorMock.Verify( + x => x.ProcessExperimentAsync(config, It.IsAny()), + Times.Never()); + } + [TestMethod] public async Task FinishAsync_SwallowsExceptionsAsync() { diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/SwiftDetectorExperimentTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/SwiftDetectorExperimentTests.cs new file mode 100644 index 000000000..363854359 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/SwiftDetectorExperimentTests.cs @@ -0,0 +1,50 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Tests.Experiments; + +using AwesomeAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.CocoaPods; +using Microsoft.ComponentDetection.Detectors.Swift; +using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class SwiftDetectorExperimentTests +{ + private readonly SwiftDetectorExperiment experiment = new(); + private readonly SwiftResolvedComponentDetector swiftDetector = new(null!, null!, null!); + private readonly PodComponentDetector podDetector = new(null!, null!, null!); + + [TestMethod] + public void IsInControlGroup_AlwaysReturnsFalse() + { + this.experiment.IsInControlGroup(this.swiftDetector).Should().BeFalse(); + this.experiment.IsInControlGroup(this.podDetector).Should().BeFalse(); + this.experiment.IsInControlGroup(Mock.Of()).Should().BeFalse(); + } + + [TestMethod] + public void IsInExperimentGroup_SwiftDetector_ReturnsTrue() + { + this.experiment.IsInExperimentGroup(this.swiftDetector).Should().BeTrue(); + } + + [TestMethod] + public void IsInExperimentGroup_OtherDetectors_ReturnsFalse() + { + this.experiment.IsInExperimentGroup(this.podDetector).Should().BeFalse(); + this.experiment.IsInExperimentGroup(Mock.Of()).Should().BeFalse(); + } + + [TestMethod] + [DataRow(0)] + [DataRow(1)] + public void ShouldRecord_AlwaysReturnsTrue(int numComponents) + { + this.experiment.ShouldRecord(this.swiftDetector, numComponents).Should().BeTrue(); + this.experiment.ShouldRecord(this.podDetector, numComponents).Should().BeTrue(); + this.experiment.ShouldRecord(Mock.Of(), numComponents).Should().BeTrue(); + } +}