Require Experimental detectors to have an ExperimentalConfiguration

docs/creating-a-new-detector.md

@@ -57,6 +57,28 @@ New detectors progress through three stages before becoming default. Contributor
 | **2. Experimental** | `IExperimentalDetector` | Runs automatically but output is NOT included in scan results. Used to measure performance impact. Enabling experimental detectors via `--DetectorArgs YourDetectorId=EnableIfDefaultOff` will cause output to be included in scan results. |
 | **3. Default** | `IComponentDetector` | Fully integrated. Runs by default and output is included in results. Can be filtered via `--DetectorCategory` or `--DetectorFilters`. |
 
+### Experimental Configurations
+
+`IExperimentalDetector` requires an `IExperimentConfiguration ExperimentConfiguration` property. Keep the
+configuration beside the detector and return the same configuration instance for the detector's lifetime.
+The experiment service discovers it from registered detectors automatically; do not register it separately.
+Configurations must include their owning detector in `IsInExperimentGroup`, or service construction fails.
+`IExperimentConfiguration` lives in the Contracts project, so detectors do not need to reference Orchestrator.
+Implement `InitAsync` for configuration initialization, returning `Task.CompletedTask` when none is needed.
+
+For comparisons against existing detectors, implement a configuration like `CondaLockDetectorExperiment`
+or `LinuxApplicationLayerExperiment`. For discoveries without an existing baseline, use
+`EmptyBaselineExperiment<TDetector>` from `Microsoft.ComponentDetection.Detectors.Experiments`:
+
+```csharp
+public IExperimentConfiguration ExperimentConfiguration { get; } =
+    new EmptyBaselineExperiment<YourEcosystemDetector>("YourEcosystemExperiment");
+```
+
+An empty-baseline experiment reports all discoveries as additions. Existing experiments retain their
+control groups and recording rules. Every experiment is supplied by a registered experimental detector;
+default-off detectors do not supply experiment configurations.
+
 ## Step 1: Define Your Component Type
 
 **Skip this step if reusing an existing component type** (e.g., `NpmComponent`, `PipComponent`).

docs/detectors/pip.md

@@ -24,6 +24,11 @@ as long as the report adheres to the following naming scheme: `component-detecti
 
 ### Legacy Detection (PipDetector, SimplePipDetector)
 
+Both legacy detectors are default-off. SimplePip was promoted to experimental in
+[#806](https://github.com/microsoft/component-detection/pull/806), then returned to default-off in
+[#836](https://github.com/microsoft/component-detection/pull/836). It has no active experiment configuration.
+Enable it explicitly with `--DetectorArgs SimplePip=EnableIfDefaultOff`.
+
 Pip detection is performed by running the following code snippet on every *setup.py*:
 
 ```python

docs/detectors/swift.md

@@ -16,7 +16,8 @@ The version, the url and commit hash of the dependencies are stored in this file
 
 ## Experiment results
 
-Swift detection is experimental. The registered `SwiftDetectorExperiment` uses an empty control group
+Swift detection is experimental. `SwiftResolvedComponentDetector.ExperimentConfiguration` supplies
+`SwiftDetectorExperiment`, which is discovered automatically and uses an empty control group
 and `SwiftResolvedComponentDetector` as its experiment group, so all Swift discoveries are reported as additions.
 It does not depend on CocoaPods detection or compare Swift packages against CocoaPods packages.
 

src/Microsoft.ComponentDetection.Contracts/IComponentDetector.cs

@@ -61,4 +61,9 @@ public interface IDefaultOffComponentDetector : IComponentDetector
 /// </summary>
 public interface IExperimentalDetector : IComponentDetector
 {
+    /// <summary>
+    /// Gets the configuration that includes this detector in its experiment group.
+    /// The configuration is discovered automatically and does not need separate service registration.
+    /// </summary>
+    IExperimentConfiguration ExperimentConfiguration { get; }
 }

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/IExperimentConfiguration.cs → src/Microsoft.ComponentDetection.Contracts/IExperimentConfiguration.cs

@@ -1,7 +1,6 @@
-namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+namespace Microsoft.ComponentDetection.Contracts;
 
 using System.Threading.Tasks;
-using Microsoft.ComponentDetection.Contracts;
 
 /// <summary>
 /// Defines the configuration for an experiment. An experiment is a set of detectors that are grouped into control and
@@ -20,7 +19,7 @@ public interface IExperimentConfiguration
     /// Initializes the experiment configuration.
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
-    Task InitAsync() => Task.CompletedTask;
+    Task InitAsync();
 
     /// <summary>
     /// Specifies if the detector is in the control group.
@@ -30,7 +29,7 @@ public interface IExperimentConfiguration
     bool IsInControlGroup(IComponentDetector componentDetector);
 
     /// <summary>
-    /// Specifies if the detector is in the control group.
+    /// Specifies if the detector is in the experiment group.
     /// </summary>
     /// <param name="componentDetector">The detector.</param>
     /// <returns><c>true</c> if the detector is in the experiment group; otherwise, <c>false</c>.</returns>

src/Microsoft.ComponentDetection.Detectors/Experiments/EmptyBaselineExperiment.cs

@@ -0,0 +1,38 @@
+namespace Microsoft.ComponentDetection.Detectors.Experiments;
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Contracts;
+
+/// <summary>
+/// Records discoveries from a detector as additions against an empty control group.
+/// </summary>
+/// <typeparam name="TDetector">The experimental detector.</typeparam>
+public class EmptyBaselineExperiment<TDetector> : IExperimentConfiguration
+    where TDetector : IExperimentalDetector
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="EmptyBaselineExperiment{TDetector}"/> class.
+    /// </summary>
+    /// <param name="name">The name of the experiment.</param>
+    public EmptyBaselineExperiment(string name)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(name);
+        this.Name = name;
+    }
+
+    /// <inheritdoc />
+    public string Name { get; }
+
+    /// <inheritdoc />
+    public Task InitAsync() => Task.CompletedTask;
+
+    /// <inheritdoc />
+    public bool IsInControlGroup(IComponentDetector componentDetector) => false;
+
+    /// <inheritdoc />
+    public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is TDetector;
+
+    /// <inheritdoc />
+    public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true;
+}

src/Microsoft.ComponentDetection.Detectors/conda/CondaLockComponentDetector.cs

@@ -26,6 +26,9 @@ public CondaLockComponentDetector(
 
     public override string Id => "CondaLock";
 
+    /// <inheritdoc />
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new CondaLockDetectorExperiment();
+
     public override IList<string> SearchPatterns { get; } = ["conda-lock.yml", "*.conda-lock.yml"];
 
     public override IEnumerable<ComponentType> SupportedComponentTypes => [ComponentType.Conda, ComponentType.Pip];

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/CondaLockDetectorExperiment.cs → src/Microsoft.ComponentDetection.Detectors/conda/CondaLockDetectorExperiment.cs

@@ -1,7 +1,7 @@
-namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+namespace Microsoft.ComponentDetection.Detectors.CondaLock;
 
+using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
-using Microsoft.ComponentDetection.Detectors.CondaLock;
 using Microsoft.ComponentDetection.Detectors.Pip;
 
 /// <summary>
@@ -12,6 +12,9 @@ public class CondaLockDetectorExperiment : IExperimentConfiguration
     /// <inheritdoc />
     public string Name => "CondaLockDetectorExperiment";
 
+    /// <inheritdoc />
+    public Task InitAsync() => Task.CompletedTask;
+
     /// <inheritdoc />
     public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is PipReportComponentDetector;
 

src/Microsoft.ComponentDetection.Detectors/dockercompose/DockerComposeComponentDetector.cs

@@ -9,6 +9,7 @@ namespace Microsoft.ComponentDetection.Detectors.DockerCompose;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.ComponentDetection.Detectors.Experiments;
 using Microsoft.Extensions.Logging;
 using YamlDotNet.RepresentationModel;
 
@@ -26,6 +27,9 @@ public DockerComposeComponentDetector(
 
     public override string Id => "DockerCompose";
 
+    /// <inheritdoc />
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new EmptyBaselineExperiment<DockerComposeComponentDetector>("DockerComposeDetectorExperiment");
+
     public override IList<string> SearchPatterns { get; } =
     [
         "docker-compose.yml", "docker-compose.yaml",

src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs

@@ -11,6 +11,7 @@ namespace Microsoft.ComponentDetection.Detectors.Dockerfile;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.ComponentDetection.Detectors.Experiments;
 using Microsoft.Extensions.Logging;
 using Sprache;
 using Valleysoft.DockerfileModel;
@@ -36,6 +37,9 @@ public DockerfileComponentDetector(
 
     public override string Id { get; } = "DockerReference";
 
+    /// <inheritdoc />
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new EmptyBaselineExperiment<DockerfileComponentDetector>("DockerfileDetectorExperiment");
+
     public override IEnumerable<string> Categories => [nameof(DetectorClass.DockerReference)];
 
     public override IList<string> SearchPatterns { get; } = ["dockerfile", "dockerfile.*", "*.dockerfile"];

src/Microsoft.ComponentDetection.Detectors/helm/HelmComponentDetector.cs

@@ -12,6 +12,7 @@ namespace Microsoft.ComponentDetection.Detectors.Helm;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.ComponentDetection.Detectors.Experiments;
 using Microsoft.Extensions.Logging;
 using YamlDotNet.RepresentationModel;
 
@@ -37,6 +38,9 @@ public HelmComponentDetector(
 
     public override string Id => "Helm";
 
+    /// <inheritdoc />
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new EmptyBaselineExperiment<HelmComponentDetector>("HelmDetectorExperiment");
+
     public override IList<string> SearchPatterns { get; } =
     [
         "Chart.yaml", "Chart.yml",

src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs

@@ -14,6 +14,7 @@ namespace Microsoft.ComponentDetection.Detectors.Ivy;
 using Microsoft.ComponentDetection.Contracts.BcdeModels;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.ComponentDetection.Detectors.Experiments;
 using Microsoft.Extensions.Logging;
 
 /// <summary>
@@ -62,6 +63,9 @@ public IvyDetector(
 
     public override string Id => "Ivy";
 
+    /// <inheritdoc />
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new EmptyBaselineExperiment<IvyDetector>("IvyDetectorExperiment");
+
     public override IList<string> SearchPatterns => ["ivy.xml"];
 
     public override IEnumerable<ComponentType> SupportedComponentTypes => [ComponentType.Maven];

src/Microsoft.ComponentDetection.Detectors/linux/LinuxApplicationLayerDetector.cs

@@ -23,6 +23,9 @@ ILogger<LinuxApplicationLayerDetector> logger
     /// <inheritdoc/>
     public new string Id => "LinuxApplicationLayer";
 
+    /// <inheritdoc/>
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new LinuxApplicationLayerExperiment();
+
     /// <inheritdoc/>
     public new IEnumerable<string> Categories =>
         [

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/LinuxApplicationLayerExperiment.cs → src/Microsoft.ComponentDetection.Detectors/linux/LinuxApplicationLayerExperiment.cs

@@ -1,7 +1,7 @@
-namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+namespace Microsoft.ComponentDetection.Detectors.Linux;
 
+using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
-using Microsoft.ComponentDetection.Detectors.Linux;
 using Microsoft.ComponentDetection.Detectors.Npm;
 using Microsoft.ComponentDetection.Detectors.NuGet;
 using Microsoft.ComponentDetection.Detectors.Pip;
@@ -20,6 +20,9 @@ public class LinuxApplicationLayerExperiment : IExperimentConfiguration
     /// <inheritdoc />
     public string Name => "LinuxApplicationLayer";
 
+    /// <inheritdoc />
+    public Task InitAsync() => Task.CompletedTask;
+
     /// <inheritdoc />
     public bool IsInControlGroup(IComponentDetector componentDetector) =>
         componentDetector

src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs

@@ -10,6 +10,7 @@ namespace Microsoft.ComponentDetection.Detectors.Poetry;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.ComponentDetection.Detectors.Experiments;
 using Microsoft.ComponentDetection.Detectors.Poetry.Contracts;
 using Microsoft.Extensions.Logging;
 using Tomlyn;
@@ -28,6 +29,9 @@ public PoetryComponentDetector(
 
     public override string Id => "Poetry";
 
+    /// <inheritdoc />
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new EmptyBaselineExperiment<PoetryComponentDetector>("PoetryDetectorExperiment");
+
     public override IList<string> SearchPatterns { get; } = ["poetry.lock"];
 
     public override IEnumerable<ComponentType> SupportedComponentTypes => [ComponentType.Pip];

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/SwiftDetectorExperiment.cs → src/Microsoft.ComponentDetection.Detectors/swiftpm/SwiftDetectorExperiment.cs

@@ -1,7 +1,7 @@
-namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+namespace Microsoft.ComponentDetection.Detectors.Swift;
 
+using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts;
-using Microsoft.ComponentDetection.Detectors.Swift;
 
 /// <summary>
 /// Records Swift Package Manager discoveries against an empty baseline.
@@ -11,6 +11,9 @@ public class SwiftDetectorExperiment : IExperimentConfiguration
     /// <inheritdoc />
     public string Name => "SwiftDetectorExperiment";
 
+    /// <inheritdoc />
+    public Task InitAsync() => Task.CompletedTask;
+
     /// <inheritdoc />
     public bool IsInControlGroup(IComponentDetector componentDetector) => false;
 

src/Microsoft.ComponentDetection.Detectors/swiftpm/SwiftResolvedComponentDetector.cs

@@ -33,6 +33,9 @@ public SwiftResolvedComponentDetector(
 
     public override string Id { get; } = "Swift";
 
+    /// <inheritdoc />
+    public IExperimentConfiguration ExperimentConfiguration { get; } = new SwiftDetectorExperiment();
+
     public override IEnumerable<string> Categories => [Enum.GetName(DetectorClass.Swift)];
 
     public override IList<string> SearchPatterns { get; } = ["Package.resolved"];

src/Microsoft.ComponentDetection.Orchestrator/Experiments/DefaultExperimentProcessor.cs

@@ -4,7 +4,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
 using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Common;
-using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Orchestrator.Experiments.Models;
 using Microsoft.Extensions.Logging;
 

src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs

@@ -9,7 +9,6 @@ namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
 using Microsoft.ComponentDetection.Common.DependencyGraph;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Orchestrator.Commands;
-using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
 using Microsoft.ComponentDetection.Orchestrator.Experiments.Models;
 using Microsoft.ComponentDetection.Orchestrator.Services;
 using Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation;
@@ -26,18 +25,28 @@ public class ExperimentService : IExperimentService
     /// <summary>
     /// Initializes a new instance of the <see cref="ExperimentService"/> class.
     /// </summary>
-    /// <param name="configs">The experiment configurations.</param>
+    /// <param name="detectors">The detectors whose experimental configurations are discovered automatically.</param>
     /// <param name="experimentProcessors">The experiment processors.</param>
     /// <param name="graphTranslationService">The graph translation service.</param>
     /// <param name="logger">The logger.</param>
     public ExperimentService(
-        IEnumerable<IExperimentConfiguration> configs,
+        IEnumerable<IComponentDetector> detectors,
         IEnumerable<IExperimentProcessor> experimentProcessors,
         IGraphTranslationService graphTranslationService,
         ILogger<ExperimentService> logger)
     {
+        var detectorConfigs = detectors.OfType<IExperimentalDetector>().Select(detector =>
+        {
+            var config = detector.ExperimentConfiguration;
+            if (config is null || !config.IsInExperimentGroup(detector))
+            {
+                throw new InvalidOperationException($"Experimental detector '{detector.Id}' must provide a configuration that includes it in the experiment group.");
+            }
+
+            return config;
+        });
         this.experiments = new ConcurrentDictionary<IExperimentConfiguration, ExperimentResults>(
-            configs.ToDictionary(config => config, _ => new ExperimentResults()));
+            detectorConfigs.Distinct().ToDictionary(config => config, _ => new ExperimentResults()));
         this.experimentProcessors = experimentProcessors;
         this.graphTranslationService = graphTranslationService;
         this.logger = logger;

src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentProcessor.cs

@@ -1,7 +1,7 @@
 namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
 
 using System.Threading.Tasks;
-using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Orchestrator.Experiments.Models;
 
 /// <summary>