Use git ls-files to find DLLs to index

Author: mbg

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -557,7 +557,7 @@ public void TestVcVarsAllBatFiles()
         [Fact]
         public void TestLinuxBuildlessExtractionSuccess()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -571,7 +571,7 @@ public void TestLinuxBuildlessExtractionSuccess()
         [Fact]
         public void TestLinuxBuildlessExtractionFailed()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 10;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -585,7 +585,7 @@ public void TestLinuxBuildlessExtractionFailed()
         [Fact]
         public void TestLinuxBuildlessExtractionSolution()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -873,7 +873,7 @@ public void TestSkipNugetMsBuild()
         [Fact]
         public void TestSkipNugetBuildless()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --skip-nuget"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -35,8 +35,6 @@ BuildScript GetCommand(string? solution)
                 if (solution is not null)
                     cmd.QuoteArgument(solution);
 
-                cmd.Argument("--references:.");
-
                 if (!builder.Options.NugetRestore)
                 {
                     cmd.Argument("--skip-nuget");

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs

@@ -15,14 +15,24 @@ internal class AssemblyCache
         /// <summary>
         /// Locate all reference files and index them.
         /// </summary>
-        /// <param name="dirs">Directories to search.</param>
+        /// <param name="paths">
+        /// Paths to search. Directories are searched recursively. Files are added directly to the
+        /// assembly cache.
+        /// </param>
         /// <param name="progressMonitor">Callback for progress.</param>
-        public AssemblyCache(IEnumerable<string> dirs, ProgressMonitor progressMonitor)
+        public AssemblyCache(IEnumerable<string> paths, ProgressMonitor progressMonitor)
         {
-            foreach (var dir in dirs)
+            foreach (var path in paths)
             {
-                progressMonitor.FindingFiles(dir);
-                AddReferenceDirectory(dir);
+                if (File.Exists(path))
+                {
+                    pendingDllsToIndex.Enqueue(path);
+                }
+                else
+                {
+                    progressMonitor.FindingFiles(path);
+                    AddReferenceDirectory(path);
+                }
             }
             IndexReferences();
         }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -31,6 +31,7 @@ public sealed class DependencyManager : IDisposable
         private readonly FileContent fileContent;
         private readonly TemporaryDirectory packageDirectory;
         private TemporaryDirectory? razorWorkingDirectory;
+        private readonly Git git;
 
 
         /// <summary>
@@ -68,7 +69,11 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 ? new[] { options.SolutionFile }
                 : allFiles.SelectFileNamesByExtension(".sln");
 
-            var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
+            // If DLL reference paths are specified on the command-line, use those to discover
+            // assemblies. Otherwise (the default), query the git CLI to determine which DLL files
+            // are tracked as part of the repository.
+            this.git = new Git(this.progressMonitor);
+            var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList();
 
             // Find DLLs in the .Net / Asp.Net Framework
             if (options.ScanNetFrameworkDlls)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// Utilities for querying information from the git CLI.
+    /// </summary>
+    internal class Git
+    {
+        private readonly ProgressMonitor progressMonitor;
+        private const string git = "git";
+
+        public Git(ProgressMonitor progressMonitor)
+        {
+            this.progressMonitor = progressMonitor;
+        }
+
+        /// <summary>
+        /// Lists all files matching <paramref name="pattern"/> which are tracked in the
+        /// current git repository.
+        /// </summary>
+        /// <param name="pattern">The file pattern.</param>
+        /// <returns>A list of all tracked files which match <paramref name="pattern"/>.</returns>
+        /// <exception cref="Exception"></exception>
+        public List<string> ListFiles(string pattern)
+        {
+            var results = new List<string>();
+            var args = string.Join(' ', "ls-files", $"\"{pattern}\"");
+
+            progressMonitor.RunningProcess($"{git} {args}");
+            var pi = new ProcessStartInfo(git, args)
+            {
+                UseShellExecute = false,
+                RedirectStandardOutput = true
+            };
+
+            using var p = new Process() { StartInfo = pi };
+            p.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
+            {
+                if (!string.IsNullOrWhiteSpace(e.Data))
+                {
+                    results.Add(e.Data);
+                }
+            });
+            p.Start();
+            p.BeginOutputReadLine();
+            p.WaitForExit();
+
+            if (p.ExitCode != 0)
+            {
+                progressMonitor.CommandFailed(git, args, p.ExitCode);
+                throw new Exception($"{git} {args} failed");
+            }
+
+            return results;
+        }
+
+    }
+}