diff --git a/docs/detectors/pnpm.md b/docs/detectors/pnpm.md index 6c8c4780f..d6ee5aa0a 100644 --- a/docs/detectors/pnpm.md +++ b/docs/detectors/pnpm.md @@ -9,6 +9,12 @@ Pnpm detection relies on the presence of lockfiles generated by the pnpm package The detector supports lockfile versions 5, 6, and 9, with version 9 being the maximum supported version. +Lockfiles can contain one or two YAML documents. When an environment document is +present, it comes before the project document. Both documents are scanned so that +config and package-manager dependencies are included alongside project dependencies. +Empty documents, more than two documents, and inconsistent declared lockfile versions +are rejected as invalid lockfiles. + ## Detection strategy Pnpm detection is performed by parsing lockfiles found in the scan directory. The `PnpmComponentDetectorFactory` acts as a version-aware factory that: diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs index ea88568dd..9db70bf9c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs @@ -14,4 +14,10 @@ internal class PnpmHasDependenciesV6 : PnpmYaml [YamlMember(Alias = "optionalDependencies")] public Dictionary OptionalDependencies { get; set; } + + [YamlMember(Alias = "packageManagerDependencies")] + public Dictionary PackageManagerDependencies { get; set; } + + [YamlMember(Alias = "configDependencies")] + public Dictionary ConfigDependencies { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs index f0e266e91..5c3d70157 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs @@ -14,4 +14,10 @@ internal class PnpmHasDependenciesV9 : PnpmYaml [YamlMember(Alias = "optionalDependencies")] public Dictionary OptionalDependencies { get; set; } + + [YamlMember(Alias = "packageManagerDependencies")] + public Dictionary PackageManagerDependencies { get; set; } + + [YamlMember(Alias = "configDependencies")] + public Dictionary ConfigDependencies { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs index da1770e5f..9ba8f8d63 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs @@ -5,17 +5,46 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; using System.Collections.Generic; using System.IO; using Microsoft.ComponentDetection.Contracts; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; using YamlDotNet.Serialization; internal abstract class PnpmParsingUtilitiesBase where T : PnpmYaml { - public T DeserializePnpmYamlFile(string fileContent) + public virtual IReadOnlyList DeserializePnpmYamlFileDocuments(string fileContent) { var deserializer = new DeserializerBuilder() .IgnoreUnmatchedProperties() .Build(); - return deserializer.Deserialize(new StringReader(fileContent)); + + using var reader = new StringReader(fileContent); + var parser = new Parser(reader); + parser.Consume(); + + var documents = new List(); + var documentCount = 0; + while (parser.TryConsume(out _)) + { + if (++documentCount > 2) + { + throw new InvalidOperationException("A pnpm lockfile must contain one or two YAML documents."); + } + + var doc = deserializer.Deserialize(parser) + ?? throw new InvalidOperationException("A pnpm lockfile must not contain empty YAML documents."); + + documents.Add(doc); + parser.TryConsume(out _); + } + + parser.Consume(); + if (documentCount == 0) + { + throw new InvalidOperationException("A pnpm lockfile must contain one or two YAML documents."); + } + + return documents.AsReadOnly(); } public virtual bool IsPnpmPackageDevDependency(Package pnpmPackage) diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs index a286c7110..3fd879cc2 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs @@ -1,8 +1,8 @@ #nullable disable namespace Microsoft.ComponentDetection.Detectors.Pnpm; -using System.IO; -using YamlDotNet.Serialization; +using System; +using System.Linq; internal static class PnpmParsingUtilitiesFactory { @@ -20,9 +20,17 @@ public static PnpmParsingUtilitiesBase Create() public static string DeserializePnpmYamlFileVersion(string fileContent) { - var deserializer = new DeserializerBuilder() - .IgnoreUnmatchedProperties() - .Build(); - return deserializer.Deserialize(new StringReader(fileContent))?.LockfileVersion; + var documents = Create().DeserializePnpmYamlFileDocuments(fileContent); + var distinctVersions = documents + .Select(doc => doc.LockfileVersion) + .Where(version => !string.IsNullOrWhiteSpace(version)) + .Distinct() + .ToList(); + if (distinctVersions.Count > 1) + { + throw new InvalidOperationException($"Inconsistent lockfile versions found: {string.Join(", ", distinctVersions)}"); + } + + return distinctVersions.FirstOrDefault(); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs index 6b34bcdd1..71c340486 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs @@ -12,36 +12,39 @@ public class Pnpm5Detector : IPnpmDetector public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder) { - var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent); + var yamls = this.pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFileContent); - foreach (var packageKeyValue in yaml?.Packages ?? Enumerable.Empty>()) + foreach (var yaml in yamls) { - // Ignore file: as these are local packages. - if (packageKeyValue.Key.StartsWith(PnpmConstants.PnpmFileDependencyPath)) + foreach (var packageKeyValue in yaml?.Packages ?? Enumerable.Empty>()) { - continue; - } + // Ignore file: as these are local packages. + if (packageKeyValue.Key.StartsWith(PnpmConstants.PnpmFileDependencyPath)) + { + continue; + } - var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: packageKeyValue.Key); - var isDevDependency = packageKeyValue.Value != null && this.pnpmParsingUtilities.IsPnpmPackageDevDependency(packageKeyValue.Value); - singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: isDevDependency); - parentDetectedComponent = singleFileComponentRecorder.GetComponent(parentDetectedComponent.Component.Id); + var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: packageKeyValue.Key); + var isDevDependency = packageKeyValue.Value != null && this.pnpmParsingUtilities.IsPnpmPackageDevDependency(packageKeyValue.Value); + singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: isDevDependency); + parentDetectedComponent = singleFileComponentRecorder.GetComponent(parentDetectedComponent.Component.Id); - if (packageKeyValue.Value.Dependencies != null) - { - foreach (var dependency in packageKeyValue.Value.Dependencies) + if (packageKeyValue.Value?.Dependencies != null) { - // Ignore local packages. - if (this.pnpmParsingUtilities.IsLocalDependency(dependency)) + foreach (var dependency in packageKeyValue.Value.Dependencies) { - continue; - } + // Ignore local packages. + if (this.pnpmParsingUtilities.IsLocalDependency(dependency)) + { + continue; + } - var childDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath( - pnpmPackagePath: this.CreatePnpmPackagePathFromDependency(dependency.Key, dependency.Value)); + var childDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath( + pnpmPackagePath: this.CreatePnpmPackagePathFromDependency(dependency.Key, dependency.Value)); - // Older code used the root's dev dependency value. We're leaving this null until we do a second pass to look at each components' top level referrers. - singleFileComponentRecorder.RegisterUsage(childDetectedComponent, parentComponentId: parentDetectedComponent.Component.Id, isDevelopmentDependency: null); + // Older code used the root's dev dependency value. We're leaving this null until we do a second pass to look at each components' top level referrers. + singleFileComponentRecorder.RegisterUsage(childDetectedComponent, parentComponentId: parentDetectedComponent.Component.Id, isDevelopmentDependency: null); + } } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs index d915c6dea..fb72d8a9d 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs @@ -12,7 +12,7 @@ public class Pnpm6Detector : IPnpmDetector public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder) { - var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent); + var yamls = this.pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFileContent); // There may be multiple instance of the same package (even at the same version) in pnpm differentiated by other aspects of the pnpm dependency path. // Therefor all DetectedComponents are tracked by the same full string pnpm uses, the pnpm dependency path, which is used as the key in this dictionary. @@ -20,25 +20,60 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom var components = new Dictionary(); // Create a component for every package referenced in the lock file. - // This includes all directly and transitively referenced dependencies. - foreach (var (pnpmDependencyPath, package) in yaml.Packages ?? Enumerable.Empty>()) + // This includes all directly and transitively referenced dependencies across all documents. + foreach (var yaml in yamls) { - // Ignore "file:" as these are local packages. - // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips). - // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references. - if (pnpmDependencyPath.StartsWith(PnpmConstants.PnpmFileDependencyPath)) + foreach (var (pnpmDependencyPath, package) in yaml.Packages ?? Enumerable.Empty>()) { - continue; + // Ignore "file:" as these are local packages. + // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips). + // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references. + if (pnpmDependencyPath.StartsWith(PnpmConstants.PnpmFileDependencyPath)) + { + continue; + } + + if (!components.TryGetValue(pnpmDependencyPath, out var existing)) + { + var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: pnpmDependencyPath); + components.Add(pnpmDependencyPath, (parentDetectedComponent, package)); + + // Register the component. + // It should get registered again with with additional information (what depended on it) later, + // but registering it now ensures nothing is missed due to a limitation in dependency traversal + // like skipping local dependencies which might have transitively depended on this. + singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package)); + } + else + { + // If the same package path occurs across multiple documents (e.g. environment and project documents), + // merge package metadata and dependencies, letting a production occurrence win for development classification. + var existingIsDev = this.pnpmParsingUtilities.IsPnpmPackageDevDependency(existing.Item2); + var newIsDev = this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package); + var effectiveIsDev = existingIsDev && newIsDev; + + if (existingIsDev && !effectiveIsDev) + { + existing.Item2.Dev = bool.FalseString; + singleFileComponentRecorder.RegisterUsage(existing.Item1, isDevelopmentDependency: false); + } + + if (package.Dependencies != null) + { + if (existing.Item2.Dependencies == null) + { + existing.Item2.Dependencies = new Dictionary(package.Dependencies); + } + else + { + foreach (var (name, version) in package.Dependencies) + { + existing.Item2.Dependencies.TryAdd(name, version); + } + } + } + } } - - var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: pnpmDependencyPath); - components.Add(pnpmDependencyPath, (parentDetectedComponent, package)); - - // Register the component. - // It should get registered again with with additional information (what depended on it) later, - // but registering it now ensures nothing is missed due to a limitation in dependency traversal - // like skipping local dependencies which might have transitively depended on this. - singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package)); } // Now that the `components` dictionary is populated, make a second pass registering all the dependency edges in the graph. @@ -56,14 +91,16 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom } // Lastly, add all direct dependencies of the current file/project setting isExplicitReferencedDependency to true: - - // "dedicated shrinkwrap" (single package) case: - this.ProcessDependencySet(singleFileComponentRecorder, components, yaml); - - // "shared shrinkwrap" (workspace / mono-repos) case: - foreach (var (_, package) in yaml.Importers ?? Enumerable.Empty>()) + foreach (var yaml in yamls) { - this.ProcessDependencySet(singleFileComponentRecorder, components, package); + // "dedicated shrinkwrap" (single package) case: + this.ProcessDependencySet(singleFileComponentRecorder, components, yaml); + + // "shared shrinkwrap" (workspace / mono-repos) case: + foreach (var (_, package) in yaml.Importers ?? Enumerable.Empty>()) + { + this.ProcessDependencySet(singleFileComponentRecorder, components, package); + } } } @@ -72,6 +109,8 @@ private void ProcessDependencySet(ISingleFileComponentRecorder singleFileCompone this.ProcessDependencyList(singleFileComponentRecorder, components, item.Dependencies); this.ProcessDependencyList(singleFileComponentRecorder, components, item.DevDependencies); this.ProcessDependencyList(singleFileComponentRecorder, components, item.OptionalDependencies); + this.ProcessDependencyList(singleFileComponentRecorder, components, item.PackageManagerDependencies); + this.ProcessDependencyList(singleFileComponentRecorder, components, item.ConfigDependencies); } private void ProcessDependencyList(ISingleFileComponentRecorder singleFileComponentRecorder, Dictionary components, Dictionary dependencies) diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm9Detector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm9Detector.cs index a275c6c3d..1ee378222 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm9Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm9Detector.cs @@ -17,7 +17,7 @@ public class Pnpm9Detector : IPnpmDetector public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder) { - var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent); + var yamls = this.pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFileContent); // There may be multiple instance of the same package (even at the same version) in pnpm differentiated by other aspects of the pnpm dependency path. // Therefor all DetectedComponents are tracked by the same full string pnpm uses, the pnpm dependency path, which is used as the key in this dictionary. @@ -25,39 +25,65 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom var components = new Dictionary(); // Create a component for every package referenced in the lock file. - // This includes all directly and transitively referenced dependencies. - foreach (var (pnpmDependencyPath, package) in yaml.Snapshots ?? Enumerable.Empty>()) + // This includes all directly and transitively referenced dependencies across all documents. + foreach (var yaml in yamls) { - // Ignore "file:" as these are local packages. - // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips). - // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references. - var (packageName, packageVersion) = this.pnpmParsingUtilities.ExtractNameAndVersionFromPnpmPackagePath(pnpmDependencyPath); - var isFileOrLink = this.IsFileOrLink(packageVersion) || this.IsFileOrLink(pnpmDependencyPath); - - var dependencyPath = pnpmDependencyPath; - if (pnpmDependencyPath.StartsWith('/')) + foreach (var (pnpmDependencyPath, package) in yaml.Snapshots ?? Enumerable.Empty>()) { - dependencyPath = pnpmDependencyPath[1..]; - } - - var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: dependencyPath); - components.Add(dependencyPath, (parentDetectedComponent, package)); - - // Register the component. - // It should get registered again with with additional information (what depended on it) later, - // but registering it now ensures nothing is missed due to a limitation in dependency traversal - // like skipping local dependencies which might have transitively depended on this. - if (!isFileOrLink) - { - singleFileComponentRecorder.RegisterUsage(parentDetectedComponent); + // Ignore "file:" as these are local packages. + // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips). + // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references. + var (packageName, packageVersion) = this.pnpmParsingUtilities.ExtractNameAndVersionFromPnpmPackagePath(pnpmDependencyPath); + var isFileOrLink = this.IsFileOrLink(packageVersion) || this.IsFileOrLink(pnpmDependencyPath); + + var dependencyPath = pnpmDependencyPath; + if (pnpmDependencyPath.StartsWith('/')) + { + dependencyPath = pnpmDependencyPath[1..]; + } + + if (!components.TryGetValue(dependencyPath, out var existing)) + { + var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: dependencyPath); + components.Add(dependencyPath, (parentDetectedComponent, package)); + + // Register the component. + // It should get registered again with with additional information (what depended on it) later, + // but registering it now ensures nothing is missed due to a limitation in dependency traversal + // like skipping local dependencies which might have transitively depended on this. + if (!isFileOrLink) + { + singleFileComponentRecorder.RegisterUsage(parentDetectedComponent); + } + } + else + { + if (package.Dependencies != null) + { + if (existing.Item2.Dependencies == null) + { + existing.Item2.Dependencies = new Dictionary(package.Dependencies); + } + else + { + foreach (var (name, version) in package.Dependencies) + { + existing.Item2.Dependencies.TryAdd(name, version); + } + } + } + } } } // now that the components dictionary is populated, add direct dependencies of the current file/project setting isExplicitReferencedDependency to true // during this step, recursively processes any indirect dependencies - foreach (var (_, package) in yaml.Importers ?? Enumerable.Empty>()) + foreach (var yaml in yamls) { - this.ProcessDependencySets(singleFileComponentRecorder, components, package); + foreach (var (_, package) in yaml.Importers ?? Enumerable.Empty>()) + { + this.ProcessDependencySets(singleFileComponentRecorder, components, package); + } } } @@ -66,6 +92,8 @@ private void ProcessDependencySets(ISingleFileComponentRecorder singleFileCompon this.ProcessDependencyList(singleFileComponentRecorder, components, item.Dependencies, isDevelopmentDependency: false); this.ProcessDependencyList(singleFileComponentRecorder, components, item.DevDependencies, isDevelopmentDependency: true); this.ProcessDependencyList(singleFileComponentRecorder, components, item.OptionalDependencies, false); + this.ProcessDependencyList(singleFileComponentRecorder, components, item.PackageManagerDependencies, false); + this.ProcessDependencyList(singleFileComponentRecorder, components, item.ConfigDependencies, false); } private void ProcessDependencyList(ISingleFileComponentRecorder singleFileComponentRecorder, Dictionary components, Dictionary dependencies, bool isDevelopmentDependency) diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs index 796016b73..9e8937f86 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetectorFactory.cs @@ -42,7 +42,7 @@ public PnpmComponentDetectorFactory( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Npm]; - public override int Version { get; } = 8; + public override int Version { get; } = 9; public override bool NeedsAutomaticRootDependencyCalculation => true; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs index de0f977e2..5dfee6a20 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs @@ -1048,4 +1048,483 @@ public async Task TestPnpmDetector_V9_GoodLockVersion_MissingSnapshotsSuccess() var detectedComponents = componentRecorder.GetDetectedComponents(); detectedComponents.Should().BeEmpty(); } + + [TestMethod] + public async Task TestPnpmDetector_V9_MultiDocumentLockfileAsync() + { + var yamlFile = @" +--- +lockfileVersion: '9.0' + +importers: + .: + configDependencies: + config-pkg: + specifier: 1.0.0 + version: 1.0.0 + packageManagerDependencies: + pnpm: + specifier: 12.3.4 + version: 12.3.4 + +packages: + config-pkg@1.0.0: + resolution: {integrity: sha512-placeholder} + pnpm@12.3.4: + resolution: {integrity: sha512-placeholder} + pnpm-dep@1.0.0: + resolution: {integrity: sha512-placeholder} + +snapshots: + config-pkg@1.0.0: {} + pnpm@12.3.4: + dependencies: + pnpm-dep: 1.0.0 + pnpm-dep@1.0.0: {} + +--- +lockfileVersion: '9.0' + +settings: + autoInstallPeers: false + +importers: + .: + dependencies: + fast-uri: + specifier: 3.1.7 + version: 3.1.7 + +packages: + fast-uri@3.1.7: + resolution: {integrity: sha512-placeholder} + uri-dep@2.0.0: + resolution: {integrity: sha512-placeholder} + +snapshots: + fast-uri@3.1.7: + dependencies: + uri-dep: 2.0.0 + uri-dep@2.0.0: {} +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(5); + var npmComponents = detectedComponents.Select(x => new { Component = (NpmComponent)x.Component, DetectedComponent = x }).ToList(); + npmComponents.Should().Contain(x => x.Component.Name == "config-pkg" && x.Component.Version == "1.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "pnpm" && x.Component.Version == "12.3.4"); + npmComponents.Should().Contain(x => x.Component.Name == "pnpm-dep" && x.Component.Version == "1.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "fast-uri" && x.Component.Version == "3.1.7"); + npmComponents.Should().Contain(x => x.Component.Name == "uri-dep" && x.Component.Version == "2.0.0"); + + var configPkg = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "config-pkg"); + var pnpm = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "pnpm"); + var pnpmDep = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "pnpm-dep"); + var fastUri = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "fast-uri"); + var uriDep = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "uri-dep"); + + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + graph.IsComponentExplicitlyReferenced(configPkg.Component.Id).Should().BeTrue(); + graph.IsComponentExplicitlyReferenced(pnpm.Component.Id).Should().BeTrue(); + graph.IsComponentExplicitlyReferenced(fastUri.Component.Id).Should().BeTrue(); + graph.GetDependenciesForComponent(pnpm.Component.Id).Should().Contain(pnpmDep.Component.Id); + graph.GetDependenciesForComponent(fastUri.Component.Id).Should().Contain(uriDep.Component.Id); + + componentRecorder.AssertAllExplicitlyReferencedComponents( + pnpmDep.Component.Id, + parent => parent.Name == "pnpm" && parent.Version == "12.3.4"); + componentRecorder.AssertAllExplicitlyReferencedComponents( + uriDep.Component.Id, + parent => parent.Name == "fast-uri" && parent.Version == "3.1.7"); + } + + [TestMethod] + public async Task TestPnpmDetector_MultiDocumentLockfile_InconsistentVersions_FailsAsync() + { + var yamlFile = @" +--- +lockfileVersion: '9.0' + +importers: + .: + dependencies: + pnpm: + specifier: 12.3.4 + version: 12.3.4 + +packages: + pnpm@12.3.4: + resolution: {integrity: sha512-placeholder} + +snapshots: + pnpm@12.3.4: {} + +--- +lockfileVersion: '6.0' + +importers: + .: + dependencies: + fast-uri: + specifier: 3.1.7 + version: 3.1.7 + +packages: + /fast-uri@3.1.7: + resolution: {integrity: sha512-placeholder} +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task TestPnpmDetector_V6_MultiDocumentLockfileAsync() + { + var yamlFile = @" +--- +lockfileVersion: '6.0' + +importers: + .: + configDependencies: + config-pkg: + specifier: 1.0.0 + version: 1.0.0 + packageManagerDependencies: + pnpm: + specifier: 12.3.4 + version: 12.3.4 + +packages: + /config-pkg@1.0.0: + resolution: {integrity: sha512-placeholder} + /pnpm@12.3.4: + resolution: {integrity: sha512-placeholder} + dependencies: + pnpm-dep: 1.0.0 + /pnpm-dep@1.0.0: + resolution: {integrity: sha512-placeholder} + +--- +lockfileVersion: '6.0' + +importers: + .: + dependencies: + fast-uri: + specifier: 3.1.7 + version: 3.1.7 + +packages: + /fast-uri@3.1.7: + resolution: {integrity: sha512-placeholder} + dependencies: + uri-dep: 2.0.0 + /uri-dep@2.0.0: + resolution: {integrity: sha512-placeholder} +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(5); + var npmComponents = detectedComponents.Select(x => new { Component = (NpmComponent)x.Component, DetectedComponent = x }).ToList(); + npmComponents.Should().Contain(x => x.Component.Name == "config-pkg" && x.Component.Version == "1.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "pnpm" && x.Component.Version == "12.3.4"); + npmComponents.Should().Contain(x => x.Component.Name == "pnpm-dep" && x.Component.Version == "1.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "fast-uri" && x.Component.Version == "3.1.7"); + npmComponents.Should().Contain(x => x.Component.Name == "uri-dep" && x.Component.Version == "2.0.0"); + + var configPkg = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "config-pkg"); + var pnpm = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "pnpm"); + var pnpmDep = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "pnpm-dep"); + var fastUri = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "fast-uri"); + var uriDep = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "uri-dep"); + + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + graph.IsComponentExplicitlyReferenced(configPkg.Component.Id).Should().BeTrue(); + graph.IsComponentExplicitlyReferenced(pnpm.Component.Id).Should().BeTrue(); + graph.IsComponentExplicitlyReferenced(fastUri.Component.Id).Should().BeTrue(); + graph.GetDependenciesForComponent(pnpm.Component.Id).Should().Contain(pnpmDep.Component.Id); + graph.GetDependenciesForComponent(fastUri.Component.Id).Should().Contain(uriDep.Component.Id); + } + + [TestMethod] + public async Task TestPnpmDetector_V6_MultiDocumentLockfile_DuplicatePackage_ProductionWinsAsync() + { + var yamlFile = @" +--- +lockfileVersion: '6.0' + +importers: + .: + devDependencies: + shared-pkg: + specifier: 1.0.0 + version: 1.0.0 + +packages: + /shared-pkg@1.0.0: + resolution: {integrity: sha512-placeholder} + dev: true + dependencies: + dep-from-doc1: 1.0.0 + /dep-from-doc1@1.0.0: + resolution: {integrity: sha512-placeholder} + dev: true + +--- +lockfileVersion: '6.0' + +importers: + .: + dependencies: + shared-pkg: + specifier: 1.0.0 + version: 1.0.0 + +packages: + /shared-pkg@1.0.0: + resolution: {integrity: sha512-placeholder} + dependencies: + dep-from-doc2: 2.0.0 + /dep-from-doc2@2.0.0: + resolution: {integrity: sha512-placeholder} +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(3); + var npmComponents = detectedComponents.Select(x => new { Component = (NpmComponent)x.Component, DetectedComponent = x }).ToList(); + npmComponents.Should().Contain(x => x.Component.Name == "shared-pkg" && x.Component.Version == "1.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "dep-from-doc1" && x.Component.Version == "1.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "dep-from-doc2" && x.Component.Version == "2.0.0"); + + var sharedPkg = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "shared-pkg"); + var dep1 = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "dep-from-doc1"); + var dep2 = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "dep-from-doc2"); + + // Production occurrence from document 2 wins over dev occurrence from document 1 + componentRecorder.GetEffectiveDevDependencyValue(sharedPkg.Component.Id).Should().BeFalse(); + + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + graph.IsComponentExplicitlyReferenced(sharedPkg.Component.Id).Should().BeTrue(); + + // Dependencies merged from both documents + var dependencies = graph.GetDependenciesForComponent(sharedPkg.Component.Id).ToList(); + dependencies.Should().Contain(dep1.Component.Id); + dependencies.Should().Contain(dep2.Component.Id); + } + + [TestMethod] + public async Task TestPnpmDetector_V5_MultiDocumentLockfileAsync() + { + var yamlFile = @" +--- +dependencies: + root-a: 1.0.0 + +packages: + /root-a/1.0.0: + dependencies: + dep-a: 1.1.0 + dev: false + /dep-a/1.1.0: + dev: false + +--- +dependencies: + root-b: 2.0.0 + +packages: + /root-b/2.0.0: + dependencies: + dep-b: 2.2.0 + dev: true + /dep-b/2.2.0: + dev: true +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("shrinkwrap.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(4); + var npmComponents = detectedComponents.Select(x => new { Component = (NpmComponent)x.Component, DetectedComponent = x }).ToList(); + npmComponents.Should().Contain(x => x.Component.Name == "root-a" && x.Component.Version == "1.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "dep-a" && x.Component.Version == "1.1.0"); + npmComponents.Should().Contain(x => x.Component.Name == "root-b" && x.Component.Version == "2.0.0"); + npmComponents.Should().Contain(x => x.Component.Name == "dep-b" && x.Component.Version == "2.2.0"); + + var rootA = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "root-a"); + var depA = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "dep-a"); + var rootB = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "root-b"); + var depB = detectedComponents.Single(c => ((NpmComponent)c.Component).Name == "dep-b"); + + componentRecorder.AssertAllExplicitlyReferencedComponents( + depA.Component.Id, + parent => parent.Name == "root-a" && parent.Version == "1.0.0"); + componentRecorder.AssertAllExplicitlyReferencedComponents( + depB.Component.Id, + parent => parent.Name == "root-b" && parent.Version == "2.0.0"); + + componentRecorder.GetEffectiveDevDependencyValue(rootA.Component.Id).GetValueOrDefault(true).Should().BeFalse(); + componentRecorder.GetEffectiveDevDependencyValue(depA.Component.Id).GetValueOrDefault(true).Should().BeFalse(); + componentRecorder.GetEffectiveDevDependencyValue(rootB.Component.Id).GetValueOrDefault(false).Should().BeTrue(); + componentRecorder.GetEffectiveDevDependencyValue(depB.Component.Id).GetValueOrDefault(false).Should().BeTrue(); + } + + [TestMethod] + [DataRow("5.0", "/present/1.0.0", "packages")] + [DataRow("6.0", "/present@1.0.0", "packages")] + [DataRow("9.0", "present@1.0.0", "snapshots")] + public async Task TestPnpmDetector_ThreeDocuments_DetectsNoComponentsAsync(string version, string packagePath, string packageSection) + { + var yamlFile = $@" +lockfileVersion: '{version}' +{packageSection}: + {packagePath}: {{}} +--- +lockfileVersion: '{version}' +--- +lockfileVersion: '{version}' +"; + + var (_, componentRecorder) = await this.detectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task TestPnpmDetector_V5_MultiDocumentLockfile_DuplicatePackage_ProductionWinsAsync(bool productionFirst) + { + var developmentDocument = @" +lockfileVersion: '5.0' +dependencies: + shared-pkg: 1.0.0 +packages: + /shared-pkg/1.0.0: + dev: true + dependencies: + dev-document-dep: 1.0.0 + /dev-document-dep/1.0.0: + dev: true +"; + var productionDocument = @" +lockfileVersion: '5.0' +dependencies: + shared-pkg: 1.0.0 +packages: + /shared-pkg/1.0.0: + dev: false + dependencies: + production-document-dep: 2.0.0 + /production-document-dep/2.0.0: + dev: false +"; + var yamlFile = productionFirst + ? productionDocument + "\n---\n" + developmentDocument + : developmentDocument + "\n---\n" + productionDocument; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents().ToList(); + components.Should().HaveCount(3); + var shared = components.Single(c => ((NpmComponent)c.Component).Name == "shared-pkg"); + var developmentDependency = components.Single(c => ((NpmComponent)c.Component).Name == "dev-document-dep"); + var productionDependency = components.Single(c => ((NpmComponent)c.Component).Name == "production-document-dep"); + + componentRecorder.GetEffectiveDevDependencyValue(shared.Component.Id).Should().BeFalse(); + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + graph.IsComponentExplicitlyReferenced(shared.Component.Id).Should().BeTrue(); + graph.GetDependenciesForComponent(shared.Component.Id).Should().BeEquivalentTo([developmentDependency.Component.Id, productionDependency.Component.Id]); + } + + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task TestPnpmDetector_V9_MultiDocumentLockfile_DuplicatePackage_ProductionWinsAsync(bool productionFirst) + { + var developmentDocument = @" +lockfileVersion: '9.0' +importers: + .: + devDependencies: + shared-pkg: + specifier: 1.0.0 + version: 1.0.0 +snapshots: + shared-pkg@1.0.0: + dependencies: + dev-document-dep: 1.0.0 + dev-document-dep@1.0.0: {} +"; + var productionDocument = @" +lockfileVersion: '9.0' +importers: + .: + dependencies: + shared-pkg: + specifier: 1.0.0 + version: 1.0.0 +snapshots: + shared-pkg@1.0.0: + dependencies: + production-document-dep: 2.0.0 + production-document-dep@2.0.0: {} +"; + var yamlFile = productionFirst + ? productionDocument + "\n---\n" + developmentDocument + : developmentDocument + "\n---\n" + productionDocument; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("pnpm-lock.yaml", yamlFile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents().ToList(); + components.Should().HaveCount(3); + var shared = components.Single(c => ((NpmComponent)c.Component).Name == "shared-pkg"); + var developmentDependency = components.Single(c => ((NpmComponent)c.Component).Name == "dev-document-dep"); + var productionDependency = components.Single(c => ((NpmComponent)c.Component).Name == "production-document-dep"); + + componentRecorder.GetEffectiveDevDependencyValue(shared.Component.Id).Should().BeFalse(); + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + graph.IsComponentExplicitlyReferenced(shared.Component.Id).Should().BeTrue(); + graph.GetDependenciesForComponent(shared.Component.Id).Should().BeEquivalentTo([developmentDependency.Component.Id, productionDependency.Component.Id]); + componentRecorder.GetEffectiveDevDependencyValue(developmentDependency.Component.Id).Should().BeFalse(); + componentRecorder.GetEffectiveDevDependencyValue(productionDependency.Component.Id).Should().BeFalse(); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs index 2965f857b..b5f111ac1 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmParsingUtilitiesTest.cs @@ -2,6 +2,8 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System; +using System.Collections.Generic; +using System.Linq; using AwesomeAssertions; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Pnpm; @@ -39,7 +41,7 @@ public void DeserializePnpmYamlFileV3() var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); var version = PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(yamlFile); version.Should().BeNull(); // Versions older than 5 report null as they don't use the same version field. - var parsedYaml = pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFile); + var parsedYaml = pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFile).Single(); parsedYaml.Packages.Should().HaveCount(2); parsedYaml.Packages.Should().ContainKey("/query-string/4.3.4"); @@ -130,7 +132,7 @@ public void DeserializePnpmYamlFileV6() var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); var version = PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(yamlFile); version.Should().Be("6.0"); - var parsedYaml = pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFile); + var parsedYaml = pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFile).Single(); parsedYaml.Packages.Should().ContainSingle(); parsedYaml.Packages.Should().ContainKey("/minimist@1.2.8"); @@ -186,4 +188,89 @@ public void ReconstructPnpmDependencyPathV6() // Absolute path: pnpmParsingUtilities.ReconstructPnpmDependencyPath("events_pkg", "/events@3.3.0").Should().BeEquivalentTo("/events@3.3.0"); } + + [TestMethod] + public void DeserializePnpmYamlFileVersion_MultiDocument_ConsistentVersions_ReturnsVersion() + { + var yamlFile = @" +--- +lockfileVersion: '9.0' +--- +lockfileVersion: '9.0' +"; + var version = PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(yamlFile); + version.Should().Be("9.0"); + } + + [TestMethod] + public void DeserializePnpmYamlFileVersion_MultiDocument_InconsistentVersions_ThrowsInvalidOperationException() + { + var yamlFile = @" +--- +lockfileVersion: '9.0' +--- +lockfileVersion: '6.0' +"; + var action = () => PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(yamlFile); + action.Should().Throw() + .WithMessage("*Inconsistent lockfile versions*"); + } + + [TestMethod] + public void DeserializePnpmYamlFileDocuments_ReturnsAllDocuments() + { + var yamlFile = @" +--- +lockfileVersion: '9.0' +importers: + .: + packageManagerDependencies: + pnpm: + specifier: 12.3.4 + version: 12.3.4 +--- +lockfileVersion: '9.0' +importers: + .: + dependencies: + fast-uri: + specifier: 3.1.7 + version: 3.1.7 +"; + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + var documents = pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFile); + + documents.Should().HaveCount(2); + documents[0].Importers["."].PackageManagerDependencies.Should().ContainKey("pnpm"); + documents[1].Importers["."].Dependencies.Should().ContainKey("fast-uri"); + } + + [TestMethod] + [DataRow("")] + [DataRow(" \n")] + [DataRow("---\n")] + [DataRow("---\n---\nlockfileVersion: '9.0'\n")] + [DataRow("lockfileVersion: '9.0'\n---\n")] + [DataRow("lockfileVersion: '9.0'\n---\nlockfileVersion: '9.0'\n---\nlockfileVersion: '9.0'\n")] + [DataRow("lockfileVersion: '9.0'\n---\n---\nlockfileVersion: '9.0'\n")] + public void DeserializePnpmYamlFileDocuments_InvalidDocumentCountOrEmptyDocument_Throws(string yamlFile) + { + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + Action parseDocuments = () => pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFile); + Action parseVersion = () => PnpmParsingUtilitiesFactory.DeserializePnpmYamlFileVersion(yamlFile); + + parseDocuments.Should().Throw(); + parseVersion.Should().Throw(); + } + + [TestMethod] + public void DeserializePnpmYamlFileDocuments_ReturnedCollectionCannotBeModified() + { + var pnpmParsingUtilities = PnpmParsingUtilitiesFactory.Create(); + var documents = pnpmParsingUtilities.DeserializePnpmYamlFileDocuments("lockfileVersion: '9.0'\n---\nlockfileVersion: '9.0'\n"); + Action removeDocument = () => ((IList)documents).RemoveAt(0); + + removeDocument.Should().Throw(); + documents.Should().HaveCount(2); + } }