From c57530070bbf4cc75e63c61470759088eccf8d7e Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 22 Jun 2026 13:34:07 +0200 Subject: [PATCH 1/7] C#: Take a the feed logic out of the try/catch for NuGet downloading. --- .../NugetPackageRestorer.cs | 63 +++++++++---------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index eb6ddd4e69bf..b5596fc66b01 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -110,48 +110,47 @@ public HashSet Restore() logger.LogInfo($"Checking NuGet feed responsiveness: {feedManager.CheckNugetFeedResponsiveness}"); compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", feedManager.CheckNugetFeedResponsiveness ? "1" : "0")); - HashSet explicitFeeds = []; HashSet reachableFeeds = []; - try - { - EmitNugetConfigDiagnostics(); + EmitNugetConfigDiagnostics(); - // Find feeds that are configured in NuGet.config files and divide them into ones that - // are explicitly configured for the project or by a private registry, and "all feeds" - // (including inherited ones) from other locations on the host outside of the working directory. - (explicitFeeds, var allFeeds) = feedManager.GetAllFeeds(); - - if (feedManager.CheckNugetFeedResponsiveness) - { - var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet(); + // Find feeds that are configured in NuGet.config files and divide them into ones that + // are explicitly configured for the project or by a private registry, and "all feeds" + // (including inherited ones) from other locations on the host outside of the working directory. + (var explicitFeeds, var allFeeds) = feedManager.GetAllFeeds(); - if (inheritedFeeds.Count > 0) - { - compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString())); - } + if (feedManager.CheckNugetFeedResponsiveness) + { + var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet(); - var timeout = feedManager.CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds); - reachableFeeds.UnionWith(reachableExplicitFeeds); + if (inheritedFeeds.Count > 0) + { + compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString())); + } - var allExplicitReachable = explicitFeeds.Count == reachableExplicitFeeds.Count; - EmitUnreachableFeedsDiagnostics(allExplicitReachable); + var timeout = feedManager.CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds); + reachableFeeds.UnionWith(reachableExplicitFeeds); - if (timeout) - { - // If we experience a timeout, we use this fallback. - // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. - var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds([], explicitFeeds); - return unresponsiveMissingPackageLocation is null - ? [] - : [unresponsiveMissingPackageLocation]; - } + var allExplicitReachable = explicitFeeds.Count == reachableExplicitFeeds.Count; + EmitUnreachableFeedsDiagnostics(allExplicitReachable); - // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). - feedManager.CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds); - reachableFeeds.UnionWith(reachableInheritedFeeds); + if (timeout) + { + // If we experience a timeout, we use this fallback. + // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. + var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds([], explicitFeeds); + return unresponsiveMissingPackageLocation is null + ? [] + : [unresponsiveMissingPackageLocation]; } + // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). + feedManager.CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds); + reachableFeeds.UnionWith(reachableInheritedFeeds); + } + + try + { using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager.IsDefaultFeedReachable)) { var count = packagesConfigRestore.InstallPackages(); From cb7e8dfed97a4ae88c90b7639b1341db0a48986d Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 22 Jun 2026 13:46:31 +0200 Subject: [PATCH 2/7] C#: Inject the feed manager into the NugetExeWrapper. --- .../NugetPackageRestorer.cs | 2 +- .../PackagesConfigRestorer.cs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index b5596fc66b01..107c4ce45f8c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -151,7 +151,7 @@ public HashSet Restore() try { - using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager.IsDefaultFeedReachable)) + using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager)) { var count = packagesConfigRestore.InstallPackages(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index 51cd27555787..42814e229278 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -33,11 +33,11 @@ internal interface IPackagesConfigRestore : IDisposable /// internal class PackagesConfigRestoreFactory { - public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, Func useDefaultFeed) + public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) { if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled()) { - return new NugetExeWrapper(fileProvider, packageDirectory, logger, useDefaultFeed); + return new NugetExeWrapper(fileProvider, packageDirectory, logger, feedManager); } return new NoOpPackagesConfig(fileProvider.PackagesConfigs, logger); @@ -65,23 +65,25 @@ private class NugetExeWrapper : IPackagesConfigRestore /// so as to not trample the source tree. /// private readonly DependencyDirectory packageDirectory; + private readonly FeedManager feedManager; private bool IsWindows => SystemBuildActions.Instance.IsWindows(); /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, Func useDefaultFeed) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; this.logger = logger; + this.feedManager = feedManager; if (fileProvider.PackagesConfigs.Count > 0) { logger.LogInfo($"Found packages.config files, trying to use nuget.exe for package restore"); nugetExe = ResolveNugetExe(); - if (!HasPackageSource() && useDefaultFeed()) + if (!HasPackageSource() && feedManager.IsDefaultFeedReachable()) { // We only modify or add a top level nuget.config file nugetConfigPath = Path.Join(fileProvider.SourceDir.FullName, "nuget.config"); From e30e0a96be99bb47610967f4cb64d1da3d70303b Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 23 Jun 2026 12:04:58 +0200 Subject: [PATCH 3/7] C#: Make the NuGetExeWrapper respect the CheckFeeds flag, private registries configuration and provide sources via the command line instead of creating a file. --- .../FeedManager.cs | 39 +++-- .../NugetPackageRestorer.cs | 18 +- .../PackagesConfigRestorer.cs | 165 +++--------------- 3 files changed, 63 insertions(+), 159 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index b9b5e16afd85..a497060bdd5a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -88,12 +88,12 @@ private IEnumerable GetFeedsFromFolder(string folderPath) => private IEnumerable GetFeedsFromNugetConfig(string nugetConfigPath) => GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath)); - private string FeedsToRestoreArgument(IEnumerable feeds) + public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArgumentPrefix) { // If there are no feeds, we want to override any default feeds that `dotnet restore` would use by passing a dummy source argument. if (!feeds.Any()) { - return $" -s \"{emptyPackageDirectory.DirInfo.FullName}\""; + return $" {sourceArgumentPrefix} \"{emptyPackageDirectory.DirInfo.FullName}\""; } // Add package sources. If any are present, they override all sources specified in @@ -101,7 +101,7 @@ private string FeedsToRestoreArgument(IEnumerable feeds) var feedArgs = new StringBuilder(); foreach (var feed in feeds) { - feedArgs.Append($" -s \"{feed}\""); + feedArgs.Append($" {sourceArgumentPrefix} \"{feed}\""); } return feedArgs.ToString(); @@ -114,15 +114,9 @@ private string FeedsToRestoreArgument(IEnumerable feeds) /// /// Path to project/solution /// The set of reachable NuGet feeds. - /// A string representing the NuGet sources argument for the restore command. - public string? MakeRestoreSourcesArgument(string path, HashSet reachableFeeds) + /// The list of NuGet feeds to use for this restore. + public IEnumerable FeedsToUse(string path, HashSet reachableFeeds) { - // Do not construct a set of explicit NuGet sources to use for restore. - if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds) - { - return null; - } - // Find the path specific feeds. var folder = GetDirectoryName(path); var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet(); @@ -136,7 +130,28 @@ private string FeedsToRestoreArgument(IEnumerable feeds) ? feedsToConsider.Where(reachableFeeds.Contains) : feedsToConsider; - return FeedsToRestoreArgument(feedsToUse); + return feedsToUse; + } + + /// + /// Constructs the list of NuGet sources to use for dotnet restore. + /// (1) Use the feeds we get from `dotnet nuget list source` + /// (2) Use private registries, if they are configured + /// + /// Path to project/solution + /// The set of reachable NuGet feeds. + /// A string representing the NuGet sources argument for the restore command. + public string? MakeDotnetRestoreSourcesArgument(string path, HashSet reachableFeeds) + { + // Do not construct a set of explicit NuGet sources to use for restore. + if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds) + { + return null; + } + + var feedsToUse = FeedsToUse(path, reachableFeeds); + + return FeedsToRestoreArgument(feedsToUse, "-s"); } private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 107c4ce45f8c..9da2018dffbc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -151,17 +151,15 @@ public HashSet Restore() try { - using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager)) + var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager, reachableFeeds); + var count = packagesConfigRestore.InstallPackages(); + if (packagesConfigRestore.PackageCount > 0) { - var count = packagesConfigRestore.InstallPackages(); - - if (packagesConfigRestore.PackageCount > 0) - { - compilationInfoContainer.CompilationInfos.Add(("packages.config files", packagesConfigRestore.PackageCount.ToString())); - compilationInfoContainer.CompilationInfos.Add(("Successfully restored packages.config files", count.ToString())); - } + compilationInfoContainer.CompilationInfos.Add(("packages.config files", packagesConfigRestore.PackageCount.ToString())); + compilationInfoContainer.CompilationInfos.Add(("Successfully restored packages.config files", count.ToString())); } + var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true }); var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet(); var excludedPaths = nugetPackageDllPaths @@ -238,7 +236,7 @@ private IEnumerable RestoreSolutions(HashSet reachableFeeds, out var projects = fileProvider.Solutions.SelectMany(solution => { logger.LogInfo($"Restoring solution {solution}..."); - var nugetSources = feedManager.MakeRestoreSourcesArgument(solution, reachableFeeds); + var nugetSources = feedManager.MakeDotnetRestoreSourcesArgument(solution, reachableFeeds); var res = dotnet.Restore(new(solution, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows)); if (res.Success) { @@ -287,7 +285,7 @@ private void RestoreProjects(IEnumerable projects, HashSet reach foreach (var project in projectGroup) { logger.LogInfo($"Restoring project {project}..."); - var nugetSources = feedManager.MakeRestoreSourcesArgument(project, reachableFeeds); + var nugetSources = feedManager.MakeDotnetRestoreSourcesArgument(project, reachableFeeds); var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows)); assets.AddDependenciesRange(res.AssetsFilePaths); lock (sync) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index 42814e229278..64ba2c5b637e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -7,7 +7,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { - internal interface IPackagesConfigRestore : IDisposable + internal interface IPackagesConfigRestore { /// /// The number of packages.config files found in the source tree. @@ -33,11 +33,11 @@ internal interface IPackagesConfigRestore : IDisposable /// internal class PackagesConfigRestoreFactory { - public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) + public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, HashSet reachableFeeds) { if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled()) { - return new NugetExeWrapper(fileProvider, packageDirectory, logger, feedManager); + return new NugetExeWrapper(fileProvider, packageDirectory, logger, feedManager, reachableFeeds); } return new NoOpPackagesConfig(fileProvider.PackagesConfigs, logger); @@ -55,8 +55,6 @@ private class NugetExeWrapper : IPackagesConfigRestore public int PackageCount => fileProvider.PackagesConfigs.Count; - private readonly string? backupNugetConfig; - private readonly string? nugetConfigPath; private readonly FileProvider fileProvider; /// @@ -66,58 +64,31 @@ private class NugetExeWrapper : IPackagesConfigRestore /// private readonly DependencyDirectory packageDirectory; private readonly FeedManager feedManager; + private readonly HashSet reachableFeeds; private bool IsWindows => SystemBuildActions.Instance.IsWindows(); + private bool? isDefaultFeedReachable; + private bool IsDefaultFeedReachable => + isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable(); + + + /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, HashSet reachableFeeds) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; this.logger = logger; this.feedManager = feedManager; + this.reachableFeeds = reachableFeeds; if (fileProvider.PackagesConfigs.Count > 0) { logger.LogInfo($"Found packages.config files, trying to use nuget.exe for package restore"); nugetExe = ResolveNugetExe(); - if (!HasPackageSource() && feedManager.IsDefaultFeedReachable()) - { - // We only modify or add a top level nuget.config file - nugetConfigPath = Path.Join(fileProvider.SourceDir.FullName, "nuget.config"); - try - { - if (File.Exists(nugetConfigPath)) - { - var tempFolderPath = FileUtils.GetTemporaryWorkingDirectory(out _); - - do - { - backupNugetConfig = Path.Join(tempFolderPath, Path.GetRandomFileName()); - } - while (File.Exists(backupNugetConfig)); - File.Copy(nugetConfigPath, backupNugetConfig, true); - } - else - { - File.WriteAllText(nugetConfigPath, - """ - - - - - - """); - } - AddDefaultPackageSource(nugetConfigPath); - } - catch (Exception e) - { - logger.LogError($"Failed to add default package source to {nugetConfigPath}: {e}"); - } - } } } @@ -200,6 +171,20 @@ private bool TryRestoreNugetPackage(string packagesConfig) { logger.LogInfo($"Restoring file \"{packagesConfig}\"..."); + var sourcesArgument = ""; + var feedsToUse = feedManager.FeedsToUse(packagesConfig, reachableFeeds).ToList(); + var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable; + + // Explicitly construct the sources to be used for the restore command if any of the following is true: + if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed) + { + if (useDefaultFeed) + { + feedsToUse.Add(FeedManager.PublicNugetOrgFeed); + } + sourcesArgument = feedManager.FeedsToRestoreArgument(feedsToUse, "-Source"); + } + /* Use nuget.exe to install a package. * Note that there is a clutch of NuGet assemblies which could be used to * invoke this directly, which would arguably be nicer. However they are @@ -210,12 +195,12 @@ private bool TryRestoreNugetPackage(string packagesConfig) if (RunWithMono) { exe = "mono"; - args = $"\"{nugetExe}\" install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\""; + args = $"\"{nugetExe}\" install -OutputDirectory \"{packageDirectory}\" {sourcesArgument} \"{packagesConfig}\""; } else { exe = nugetExe!; - args = $"install -OutputDirectory \"{packageDirectory}\" \"{packagesConfig}\""; + args = $"install -OutputDirectory \"{packageDirectory}\" {sourcesArgument} \"{packagesConfig}\""; } var pi = new ProcessStartInfo(exe, args) @@ -248,98 +233,6 @@ public int InstallPackages() { return fileProvider.PackagesConfigs.Count(TryRestoreNugetPackage); } - - private bool HasPackageSource() - { - if (IsWindows) - { - return true; - } - - try - { - logger.LogInfo("Checking if default package source is available..."); - RunMonoNugetCommand("sources list -ForceEnglishOutput", out var stdout); - if (stdout.All(line => line != "No sources found.")) - { - return true; - } - - return false; - } - catch (Exception e) - { - logger.LogWarning($"Failed to check if default package source is added: {e}"); - return true; - } - } - - private void RunMonoNugetCommand(string command, out IList stdout) - { - string exe, args; - if (RunWithMono) - { - exe = "mono"; - args = $"\"{nugetExe}\" {command}"; - } - else - { - exe = nugetExe!; - args = command; - } - - var pi = new ProcessStartInfo(exe, args) - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false - }; - - var threadId = Environment.CurrentManagedThreadId; - void onOut(string s) => logger.LogDebug(s, threadId); - void onError(string s) => logger.LogError(s, threadId); - pi.ReadOutput(out stdout, onOut, onError); - } - - private void AddDefaultPackageSource(string nugetConfig) - { - logger.LogInfo("Adding default package source..."); - RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {FeedManager.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out _); - } - - public void Dispose() - { - if (nugetConfigPath is null) - { - return; - } - - try - { - if (backupNugetConfig is null) - { - logger.LogInfo("Removing nuget.config file"); - File.Delete(nugetConfigPath); - return; - } - - logger.LogInfo("Reverting nuget.config file content"); - // The content of the original nuget.config file is reverted without changing the file's attributes or casing: - using (var backup = File.OpenRead(backupNugetConfig)) - using (var current = File.OpenWrite(nugetConfigPath)) - { - current.SetLength(0); // Truncate file - backup.CopyTo(current); // Restore original content - } - - logger.LogInfo("Deleting backup nuget.config file"); - File.Delete(backupNugetConfig); - } - catch (Exception exc) - { - logger.LogError($"Failed to restore original nuget.config file: {exc}"); - } - } } private class NoOpPackagesConfig : IPackagesConfigRestore @@ -363,8 +256,6 @@ public int InstallPackages() } return 0; } - - public void Dispose() { } } } } From 9bce72a8ef8945a78c9ecc2a4fa1e3dc486cf28c Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 24 Jun 2026 14:22:50 +0200 Subject: [PATCH 4/7] C#: Address review comments. --- .../FeedManager.cs | 4 ++-- .../PackagesConfigRestorer.cs | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index a497060bdd5a..744b60f3d3f5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -90,7 +90,7 @@ private IEnumerable GetFeedsFromNugetConfig(string nugetConfigPath) => public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArgumentPrefix) { - // If there are no feeds, we want to override any default feeds that `dotnet restore` would use by passing a dummy source argument. + // If there are no feeds, we want to override any default feeds that `restore` would use by passing a dummy source argument. if (!feeds.Any()) { return $" {sourceArgumentPrefix} \"{emptyPackageDirectory.DirInfo.FullName}\""; @@ -112,7 +112,7 @@ public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArg /// (1) Use the feeds we get from `dotnet nuget list source` /// (2) Use private registries, if they are configured /// - /// Path to project/solution + /// Path to project/solution/packages.config /// The set of reachable NuGet feeds. /// The list of NuGet feeds to use for this restore. public IEnumerable FeedsToUse(string path, HashSet reachableFeeds) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index 64ba2c5b637e..af484ba406e4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -72,8 +72,6 @@ private class NugetExeWrapper : IPackagesConfigRestore private bool IsDefaultFeedReachable => isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable(); - - /// /// Create the package manager for a specified source tree. /// @@ -175,7 +173,8 @@ private bool TryRestoreNugetPackage(string packagesConfig) var feedsToUse = feedManager.FeedsToUse(packagesConfig, reachableFeeds).ToList(); var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable; - // Explicitly construct the sources to be used for the restore command if any of the following is true: + // Explicitly construct the sources to be used for the restore command when checking feed + // responsiveness, using private registries, or falling back to nuget.org. if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed) { if (useDefaultFeed) From e3cf5c870a957904c8a85e625e2d22b6ff589239 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 24 Jun 2026 15:46:23 +0200 Subject: [PATCH 5/7] C#: Add change-note. --- .../ql/lib/change-notes/2026-06-24-nuget-packages-config.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 csharp/ql/lib/change-notes/2026-06-24-nuget-packages-config.md diff --git a/csharp/ql/lib/change-notes/2026-06-24-nuget-packages-config.md b/csharp/ql/lib/change-notes/2026-06-24-nuget-packages-config.md new file mode 100644 index 000000000000..5b236a118da7 --- /dev/null +++ b/csharp/ql/lib/change-notes/2026-06-24-nuget-packages-config.md @@ -0,0 +1,4 @@ +--- +category: majorAnalysis +--- +* Simplified and streamlined the use of NuGet sources when downloading dependencies via `[mono] nuget.exe` in `build-mode: none`: NuGet sources are now supplied via the `-Source` flag instead of moving or creating `nuget.config` files in the checked-out repository, private registries are used if configured, and only reachable feeds are used when NuGet feed checking is enabled (the default). From c508d424cf5eff714b1681299a12c7fb93ee7409 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 24 Jun 2026 10:56:07 +0200 Subject: [PATCH 6/7] C#: Re-factor GetAllFeeds into properties in the Feed Manager and encapsulate as immutable hash sets. --- .../FeedManager.cs | 26 +++++++++++++++---- .../NugetPackageRestorer.cs | 7 ++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 744b60f3d3f5..1dc4c2ac1666 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -29,6 +29,12 @@ internal sealed partial class FeedManager : IDisposable public bool HasPrivateRegistryFeeds { get; } public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness); + private readonly Lazy> lazyExplicitFeeds; + public ImmutableHashSet ExplicitFeeds => lazyExplicitFeeds.Value; + + private readonly Lazy> lazyAllFeeds; + public ImmutableHashSet AllFeeds => lazyAllFeeds.Value; + public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider) { this.logger = logger; @@ -38,6 +44,9 @@ public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotPr PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); + + lazyExplicitFeeds = new Lazy>(GetExplicitFeeds); + lazyAllFeeds = new Lazy>(GetAllFeeds); } private string? GetDirectoryName(string path) @@ -271,7 +280,7 @@ private HashSet GetExcludedFeeds() /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured /// to be excluded from the check) or false otherwise. /// - public bool CheckSpecifiedFeeds(HashSet feeds, out HashSet reachableFeeds) + public bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out HashSet reachableFeeds) { // Exclude any feeds from the feed check that are configured by the corresponding environment variable. // These feeds are always assumed to be reachable. @@ -342,7 +351,7 @@ private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool i return reachableFeeds; } - public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNugetConfigs) + public List GetReachableFallbackNugetFeeds(ImmutableHashSet? feedsFromNugetConfigs) { var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); if (fallbackFeeds.Count == 0) @@ -365,7 +374,7 @@ public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNug return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _); } - public (HashSet explicitFeeds, HashSet allFeeds) GetAllFeeds() + private ImmutableHashSet GetExplicitFeeds() { var nugetConfigs = fileProvider.NugetConfigs; @@ -391,10 +400,17 @@ public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNug explicitFeeds.UnionWith(PrivateRegistryFeeds); } + return explicitFeeds.ToImmutableHashSet(); + } + + private ImmutableHashSet GetAllFeeds() + { + var nugetConfigs = fileProvider.NugetConfigs; + HashSet allFeeds = []; // Add all explicitFeeds to the set of all feeds. - allFeeds.UnionWith(explicitFeeds); + allFeeds.UnionWith(ExplicitFeeds); // Obtain the list of feeds from the root source directory. // If a NuGet file is present it will be respected, otherwise we will just get the machine/environment specific feeds. @@ -414,7 +430,7 @@ public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNug logger.LogInfo($"Found {allFeeds.Count} NuGet feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}"); - return (explicitFeeds, allFeeds); + return allFeeds.ToImmutableHashSet(); } [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 9da2018dffbc..2b55923e46b3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -117,11 +117,12 @@ public HashSet Restore() // Find feeds that are configured in NuGet.config files and divide them into ones that // are explicitly configured for the project or by a private registry, and "all feeds" // (including inherited ones) from other locations on the host outside of the working directory. - (var explicitFeeds, var allFeeds) = feedManager.GetAllFeeds(); + var explicitFeeds = feedManager.ExplicitFeeds; + var allFeeds = feedManager.AllFeeds; if (feedManager.CheckNugetFeedResponsiveness) { - var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet(); + var inheritedFeeds = allFeeds.Except(explicitFeeds).ToImmutableHashSet(); if (inheritedFeeds.Count > 0) { @@ -312,7 +313,7 @@ private void RestoreProjects(IEnumerable projects, HashSet reach compilationInfoContainer.CompilationInfos.Add(("Failed project restore with missing package error", nugetMissingPackageFailures.ToString())); } - private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable usedPackageNames, HashSet? feedsFromNugetConfigs) + private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable usedPackageNames, ImmutableHashSet? feedsFromNugetConfigs) { var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(feedsFromNugetConfigs); compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); From 036f75ca90eba07c16004aad64f29b7cc1e870f8 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 24 Jun 2026 12:56:36 +0200 Subject: [PATCH 7/7] C#: Re-factor more feed sets into the feed manager (and further use immutable hash sets). --- .../FeedManager.cs | 70 ++++++++++++++++--- .../NugetPackageRestorer.cs | 20 ++---- .../PackagesConfigRestorer.cs | 7 +- 3 files changed, 70 insertions(+), 27 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 1dc4c2ac1666..b0a402148220 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -24,31 +24,79 @@ internal sealed partial class FeedManager : IDisposable private readonly FileProvider fileProvider; private readonly DependabotProxy? dependabotProxy; private readonly DependencyDirectory emptyPackageDirectory; + private readonly ImmutableHashSet privateRegistryFeeds; - public ImmutableHashSet PrivateRegistryFeeds { get; } public bool HasPrivateRegistryFeeds { get; } public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness); private readonly Lazy> lazyExplicitFeeds; + + /// + /// Gets the list of NuGet feeds that are explicitly configured + /// - NuGet configuration files. + /// - Private package registries that are configured for C#. + /// public ImmutableHashSet ExplicitFeeds => lazyExplicitFeeds.Value; private readonly Lazy> lazyAllFeeds; + + /// + /// Gets the list of all NuGet feeds that are configured in the environment. That is + /// - Explicit feeds + /// - Inherited feeds from the machine and environment (if not explicitly disabled by a + /// root directory NuGet configuration). + /// public ImmutableHashSet AllFeeds => lazyAllFeeds.Value; + /// + /// Gets the list of inherited NuGet feeds that are configured in the environment. + /// + public ImmutableHashSet InheritedFeeds => AllFeeds.Except(ExplicitFeeds).ToImmutableHashSet(); + + private readonly Lazy<(bool, ImmutableHashSet)> lazyReachableExplicitFeeds; + + /// + /// Gets whether there was a timeout when checking the reachability of the explicitly configured NuGet feeds. + /// + public bool ExplicitFeedTimeout => lazyReachableExplicitFeeds.Value.Item1; + + /// + /// Gets the list of reachable NuGet feeds that are explicitly configured. + /// + public ImmutableHashSet ReachableExplicitFeeds => lazyReachableExplicitFeeds.Value.Item2; + + private readonly Lazy> lazyReachableFeeds; + /// + /// Gets the list of reachable NuGet feeds that are configured in the environment. + /// + public ImmutableHashSet ReachableFeeds => lazyReachableFeeds.Value; + public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider) { this.logger = logger; this.dotnet = dotnet; this.dependabotProxy = dependabotProxy; this.fileProvider = fileProvider; - PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; - HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0; + privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; + HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); lazyExplicitFeeds = new Lazy>(GetExplicitFeeds); lazyAllFeeds = new Lazy>(GetAllFeeds); + lazyReachableExplicitFeeds = new Lazy<(bool, ImmutableHashSet)>(() => + { + var timeout = CheckSpecifiedFeeds(ExplicitFeeds, out var reachableFeeds); + return (timeout, reachableFeeds); + }); + lazyReachableFeeds = new Lazy>(() => + { + // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). + CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds); + return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet(); + }); } + private string? GetDirectoryName(string path) { try @@ -124,7 +172,7 @@ public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArg /// Path to project/solution/packages.config /// The set of reachable NuGet feeds. /// The list of NuGet feeds to use for this restore. - public IEnumerable FeedsToUse(string path, HashSet reachableFeeds) + public IEnumerable FeedsToUse(string path, ImmutableHashSet reachableFeeds) { // Find the path specific feeds. var folder = GetDirectoryName(path); @@ -132,7 +180,7 @@ public IEnumerable FeedsToUse(string path, HashSet reachableFeed if (HasPrivateRegistryFeeds) { - feedsToConsider.UnionWith(PrivateRegistryFeeds); + feedsToConsider.UnionWith(privateRegistryFeeds); } var feedsToUse = CheckNugetFeedResponsiveness @@ -150,7 +198,7 @@ public IEnumerable FeedsToUse(string path, HashSet reachableFeed /// Path to project/solution /// The set of reachable NuGet feeds. /// A string representing the NuGet sources argument for the restore command. - public string? MakeDotnetRestoreSourcesArgument(string path, HashSet reachableFeeds) + public string? MakeDotnetRestoreSourcesArgument(string path, ImmutableHashSet reachableFeeds) { // Do not construct a set of explicit NuGet sources to use for restore. if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds) @@ -280,7 +328,7 @@ private HashSet GetExcludedFeeds() /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured /// to be excluded from the check) or false otherwise. /// - public bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out HashSet reachableFeeds) + private bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out ImmutableHashSet reachableFeeds) { // Exclude any feeds from the feed check that are configured by the corresponding environment variable. // These feeds are always assumed to be reachable. @@ -296,10 +344,10 @@ public bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out HashSet excludedFeeds.Contains(feed))); + reachableFeeds = reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet(); return isTimeout; } @@ -396,8 +444,8 @@ private ImmutableHashSet GetExplicitFeeds() // in addition to the ones that are configured in `nuget.config` files. if (HasPrivateRegistryFeeds) { - logger.LogInfo($"Found {PrivateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", PrivateRegistryFeeds.OrderBy(f => f))}"); - explicitFeeds.UnionWith(PrivateRegistryFeeds); + logger.LogInfo($"Found {privateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", privateRegistryFeeds.OrderBy(f => f))}"); + explicitFeeds.UnionWith(privateRegistryFeeds); } return explicitFeeds.ToImmutableHashSet(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 2b55923e46b3..98d5394b9235 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -110,7 +110,7 @@ public HashSet Restore() logger.LogInfo($"Checking NuGet feed responsiveness: {feedManager.CheckNugetFeedResponsiveness}"); compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", feedManager.CheckNugetFeedResponsiveness ? "1" : "0")); - HashSet reachableFeeds = []; + ImmutableHashSet reachableFeeds = []; EmitNugetConfigDiagnostics(); @@ -118,24 +118,20 @@ public HashSet Restore() // are explicitly configured for the project or by a private registry, and "all feeds" // (including inherited ones) from other locations on the host outside of the working directory. var explicitFeeds = feedManager.ExplicitFeeds; - var allFeeds = feedManager.AllFeeds; if (feedManager.CheckNugetFeedResponsiveness) { - var inheritedFeeds = allFeeds.Except(explicitFeeds).ToImmutableHashSet(); + var inheritedFeeds = feedManager.InheritedFeeds; if (inheritedFeeds.Count > 0) { compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString())); } - var timeout = feedManager.CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds); - reachableFeeds.UnionWith(reachableExplicitFeeds); - - var allExplicitReachable = explicitFeeds.Count == reachableExplicitFeeds.Count; + var allExplicitReachable = explicitFeeds.Count == feedManager.ReachableExplicitFeeds.Count; EmitUnreachableFeedsDiagnostics(allExplicitReachable); - if (timeout) + if (feedManager.ExplicitFeedTimeout) { // If we experience a timeout, we use this fallback. // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. @@ -145,9 +141,7 @@ public HashSet Restore() : [unresponsiveMissingPackageLocation]; } - // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). - feedManager.CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds); - reachableFeeds.UnionWith(reachableInheritedFeeds); + reachableFeeds = feedManager.ReachableFeeds; } try @@ -224,7 +218,7 @@ public HashSet Restore() /// Populates dependencies with the relevant dependencies from the assets files generated by the restore. /// Returns a list of projects that are up to date with respect to restore. /// - private IEnumerable RestoreSolutions(HashSet reachableFeeds, out DependencyContainer dependencies) + private IEnumerable RestoreSolutions(ImmutableHashSet reachableFeeds, out DependencyContainer dependencies) { var successCount = 0; var nugetSourceFailures = 0; @@ -269,7 +263,7 @@ private IEnumerable RestoreSolutions(HashSet reachableFeeds, out /// /// A list of paths to project files. /// The set of reachable NuGet feeds. - private void RestoreProjects(IEnumerable projects, HashSet reachableFeeds, out ConcurrentBag dependencies) + private void RestoreProjects(IEnumerable projects, ImmutableHashSet reachableFeeds, out ConcurrentBag dependencies) { var successCount = 0; var nugetSourceFailures = 0; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index af484ba406e4..26f9e42d435d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.IO; using System.Linq; @@ -33,7 +34,7 @@ internal interface IPackagesConfigRestore /// internal class PackagesConfigRestoreFactory { - public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, HashSet reachableFeeds) + public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, ImmutableHashSet reachableFeeds) { if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled()) { @@ -64,7 +65,7 @@ private class NugetExeWrapper : IPackagesConfigRestore /// private readonly DependencyDirectory packageDirectory; private readonly FeedManager feedManager; - private readonly HashSet reachableFeeds; + private readonly ImmutableHashSet reachableFeeds; private bool IsWindows => SystemBuildActions.Instance.IsWindows(); @@ -75,7 +76,7 @@ private class NugetExeWrapper : IPackagesConfigRestore /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, HashSet reachableFeeds) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, ImmutableHashSet reachableFeeds) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory;