Merge pull request #16164 from tamasvajk/buildless/nuget-feed-fallback-feed-check

C#: Check fallback nuget feeds before trying to use them in the fallb…

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -339,7 +339,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
                 {
                     logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
 
-                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
+                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
                     {
                         runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
                     }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -95,9 +95,9 @@ public bool AddPackage(string folder, string package)
 
         public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
-        private IList<string> GetResultList(string args)
+        private IList<string> GetResultList(string args, string? workingDirectory = null)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var results))
+            if (dotnetCliInvoker.RunCommand(args, workingDirectory, out var results))
             {
                 return results;
             }
@@ -111,7 +111,19 @@ public bool Exec(string execArgs)
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
+        private const string nugetListSourceCommand = "nuget list source --format Short";
+
+        public IList<string> GetNugetFeeds(string nugetConfig)
+        {
+            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
+            return GetResultList($"{nugetListSourceCommand} --configfile \"{nugetConfig}\"");
+        }
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath)
+        {
+            logger.LogInfo($"Getting Nuget feeds in folder '{folderPath}'...");
+            return GetResultList(nugetListSourceCommand, folderPath);
+        }
 
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -21,41 +21,49 @@ public DotNetCliInvoker(ILogger logger, string exec)
             this.Exec = exec;
         }
 
-        private ProcessStartInfo MakeDotnetStartInfo(string args)
+        private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory)
         {
             var startInfo = new ProcessStartInfo(Exec, args)
             {
                 UseShellExecute = false,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true
             };
+            if (!string.IsNullOrWhiteSpace(workingDirectory))
+            {
+                startInfo.WorkingDirectory = workingDirectory;
+            }
             // Set the .NET CLI language to English to avoid localized output.
             startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
             startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
             startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";
             return startInfo;
         }
 
-        private bool RunCommandAux(string args, out IList<string> output)
+        private bool RunCommandAux(string args, string? workingDirectory, out IList<string> output)
         {
-            logger.LogInfo($"Running {Exec} {args}");
-            var pi = MakeDotnetStartInfo(args);
+            var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}";
+            logger.LogInfo($"Running {Exec} {args}{dirLog}");
+            var pi = MakeDotnetStartInfo(args, workingDirectory);
             var threadId = Environment.CurrentManagedThreadId;
             void onOut(string s) => logger.LogInfo(s, threadId);
             void onError(string s) => logger.LogError(s, threadId);
             var exitCode = pi.ReadOutput(out output, onOut, onError);
             if (exitCode != 0)
             {
-                logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
+                logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}");
                 return false;
             }
             return true;
         }
 
         public bool RunCommand(string args) =>
-            RunCommandAux(args, out _);
+            RunCommandAux(args, null, out _);
 
         public bool RunCommand(string args, out IList<string> output) =>
-            RunCommandAux(args, out output);
+            RunCommandAux(args, null, out output);
+
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output) =>
+            RunCommandAux(args, workingDirectory, out output);
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs

@@ -33,10 +33,28 @@ internal class EnvironmentVariableNames
         public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
 
         /// <summary>
-        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
+        /// Specifies the timeout (as an integer) in milliseconds for the initial check of fallback NuGet feeds responsiveness. The value is then doubled for each subsequent check.
+        /// This is primarily used in testing.
+        /// </summary>
+        internal const string NugetFeedResponsivenessInitialTimeoutForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT";
+
+        /// <summary>
+        /// Specifies how many requests to make to the NuGet feeds to check their responsiveness.
         /// </summary>
         public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
 
+        /// <summary>
+        /// Specifies how many requests to make to the fallback NuGet feeds to check their responsiveness.
+        /// This is primarily used in testing.
+        /// </summary>
+        internal const string NugetFeedResponsivenessRequestCountForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT";
+
+        /// <summary>
+        /// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs.
+        /// The default value is `https://api.nuget.org/v3/index.json`.
+        /// </summary>
+        public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
+
         /// <summary>
         /// Specifies the location of the diagnostic directory.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -14,6 +14,7 @@ public interface IDotNet
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
         IList<string> GetNugetFeeds(string nugetConfig);
+        IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -19,5 +19,11 @@ internal interface IDotNetCliInvoker
         /// The output of the command is returned in `output`.
         /// </summary>
         bool RunCommand(string args, out IList<string> output);
+
+        /// <summary>
+        /// Execute `dotnet <args>` in `<workingDirectory>` and return true if the command succeeded, otherwise false.
+        /// The output of the command is returned in `output`.
+        /// </summary>
+        bool RunCommand(string args, string? workingDirectory, out IList<string> output);
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs

@@ -243,7 +243,7 @@ private void RunMonoNugetCommand(string command, out IList<string> stdout)
         private void AddDefaultPackageSource(string nugetConfig)
         {
             logger.LogInfo("Adding default package source...");
-            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);
+            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _);
         }
 
         public void Dispose()

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs

@@ -10,6 +10,7 @@ internal class DotNetCliInvokerStub : IDotNetCliInvoker
     {
         private readonly IList<string> output;
         private string lastArgs = "";
+        public string WorkingDirectory { get; private set; } = "";
         public bool Success { get; set; } = true;
 
         public DotNetCliInvokerStub(IList<string> output)
@@ -32,6 +33,12 @@ public bool RunCommand(string args, out IList<string> output)
             return Success;
         }
 
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output)
+        {
+            WorkingDirectory = workingDirectory ?? "";
+            return RunCommand(args, out output);
+        }
+
         public string GetLastArgs() => lastArgs;
     }
 
@@ -262,5 +269,36 @@ public void TestDotnetExec()
             var lastArgs = dotnetCliInvoker.GetLastArgs();
             Assert.Equal("exec myarg1 myarg2", lastArgs);
         }
+
+        [Fact]
+        public void TestNugetFeeds()
+        {
+            // Setup
+            var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
+            var dotnet = MakeDotnet(dotnetCliInvoker);
+
+            // Execute
+            dotnet.GetNugetFeeds("abc");
+
+            // Verify
+            var lastArgs = dotnetCliInvoker.GetLastArgs();
+            Assert.Equal("nuget list source --format Short --configfile \"abc\"", lastArgs);
+        }
+
+        [Fact]
+        public void TestNugetFeedsFromFolder()
+        {
+            // Setup
+            var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
+            var dotnet = MakeDotnet(dotnetCliInvoker);
+
+            // Execute
+            dotnet.GetNugetFeedsFromFolder("abc");
+
+            // Verify
+            var lastArgs = dotnetCliInvoker.GetLastArgs();
+            Assert.Equal("nuget list source --format Short", lastArgs);
+            Assert.Equal("abc", dotnetCliInvoker.WorkingDirectory);
+        }
     }
 }

csharp/extractor/Semmle.Extraction.Tests/DotNet.cs

@@ -28,6 +28,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
         public bool Exec(string execArgs) => true;
 
         public IList<string> GetNugetFeeds(string nugetConfig) => [];
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath) => [];
     }
 
     public class RuntimeTests