Skip to content

C#: Improve package download logic. #22041

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
michaelnebel wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

C#: Improve package download logic. #22041

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c575300
C#: Take a the feed logic out of the try/catch for NuGet downloading.
michaelnebel Jun 22, 2026
cb7e8df
C#: Inject the feed manager into the NugetExeWrapper.
michaelnebel Jun 22, 2026
e30e0a9
C#: Make the NuGetExeWrapper respect the CheckFeeds flag, private reg…
michaelnebel Jun 23, 2026
9bce72a
C#: Address review comments.
michaelnebel Jun 24, 2026
e3cf5c8
C#: Add change-note.
michaelnebel Jun 24, 2026
c508d42
C#: Re-factor GetAllFeeds into properties in the Feed Manager and enc…
michaelnebel Jun 24, 2026
036f75c
C#: Re-factor more feed sets into the feed manager (and further use i…
michaelnebel Jun 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 106 additions & 27 deletions csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,79 @@ internal sealed partial class FeedManager : IDisposable
private readonly FileProvider fileProvider;
private readonly DependabotProxy? dependabotProxy;
private readonly DependencyDirectory emptyPackageDirectory;
private readonly ImmutableHashSet<string> privateRegistryFeeds;

public ImmutableHashSet<string> PrivateRegistryFeeds { get; }
public bool HasPrivateRegistryFeeds { get; }
public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness);

private readonly Lazy<ImmutableHashSet<string>> lazyExplicitFeeds;

/// <summary>
/// Gets the list of NuGet feeds that are explicitly configured
/// - NuGet configuration files.
/// - Private package registries that are configured for C#.
/// </summary>
public ImmutableHashSet<string> ExplicitFeeds => lazyExplicitFeeds.Value;

private readonly Lazy<ImmutableHashSet<string>> lazyAllFeeds;

/// <summary>
/// Gets the list of all NuGet feeds that are configured in the environment. That is
/// - Explicit feeds
/// - Inherited feeds from the machine and environment (if not explicitly disabled by a
/// root directory NuGet configuration).
/// </summary>
public ImmutableHashSet<string> AllFeeds => lazyAllFeeds.Value;

/// <summary>
/// Gets the list of inherited NuGet feeds that are configured in the environment.
/// </summary>
public ImmutableHashSet<string> InheritedFeeds => AllFeeds.Except(ExplicitFeeds).ToImmutableHashSet();

private readonly Lazy<(bool, ImmutableHashSet<string>)> lazyReachableExplicitFeeds;

/// <summary>
/// Gets whether there was a timeout when checking the reachability of the explicitly configured NuGet feeds.
/// </summary>
public bool ExplicitFeedTimeout => lazyReachableExplicitFeeds.Value.Item1;

/// <summary>
/// Gets the list of reachable NuGet feeds that are explicitly configured.
/// </summary>
public ImmutableHashSet<string> ReachableExplicitFeeds => lazyReachableExplicitFeeds.Value.Item2;

private readonly Lazy<ImmutableHashSet<string>> lazyReachableFeeds;
/// <summary>
/// Gets the list of reachable NuGet feeds that are configured in the environment.
/// </summary>
public ImmutableHashSet<string> ReachableFeeds => lazyReachableFeeds.Value;

public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider)
{
this.logger = logger;
this.dotnet = dotnet;
this.dependabotProxy = dependabotProxy;
this.fileProvider = fileProvider;
PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0;
privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);

lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
lazyAllFeeds = new Lazy<ImmutableHashSet<string>>(GetAllFeeds);
lazyReachableExplicitFeeds = new Lazy<(bool, ImmutableHashSet<string>)>(() =>
{
var timeout = CheckSpecifiedFeeds(ExplicitFeeds, out var reachableFeeds);
return (timeout, reachableFeeds);
});
lazyReachableFeeds = new Lazy<ImmutableHashSet<string>>(() =>
{
// Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific).
CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds);
return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet();
});
}


private string? GetDirectoryName(string path)
{
try
Expand Down Expand Up @@ -88,20 +145,20 @@ private IEnumerable<string> GetFeedsFromFolder(string folderPath) =>
private IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath));

private string FeedsToRestoreArgument(IEnumerable<string> feeds)
public string FeedsToRestoreArgument(IEnumerable<string> feeds, string sourceArgumentPrefix)
{
// If there are no feeds, we want to override any default feeds that `dotnet restore` would use by passing a dummy source argument.
// If there are no feeds, we want to override any default feeds that `restore` would use by passing a dummy source argument.
if (!feeds.Any())
{
return $" -s \"{emptyPackageDirectory.DirInfo.FullName}\"";
return $" {sourceArgumentPrefix} \"{emptyPackageDirectory.DirInfo.FullName}\"";
}

// Add package sources. If any are present, they override all sources specified in
// the configuration file(s).
var feedArgs = new StringBuilder();
foreach (var feed in feeds)
{
feedArgs.Append($" -s \"{feed}\"");
feedArgs.Append($" {sourceArgumentPrefix} \"{feed}\"");
}

return feedArgs.ToString();
Expand All @@ -112,31 +169,46 @@ private string FeedsToRestoreArgument(IEnumerable<string> feeds)
/// (1) Use the feeds we get from `dotnet nuget list source`
/// (2) Use private registries, if they are configured
/// </summary>
/// <param name="path">Path to project/solution</param>
/// <param name="path">Path to project/solution/packages.config</param>
/// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
/// <returns>A string representing the NuGet sources argument for the restore command.</returns>
public string? MakeRestoreSourcesArgument(string path, HashSet<string> reachableFeeds)
/// <returns>The list of NuGet feeds to use for this restore.</returns>
public IEnumerable<string> FeedsToUse(string path, ImmutableHashSet<string> reachableFeeds)
{
// Do not construct a set of explicit NuGet sources to use for restore.
if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
{
return null;
}

// Find the path specific feeds.
var folder = GetDirectoryName(path);
var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet<string>();

if (HasPrivateRegistryFeeds)
{
feedsToConsider.UnionWith(PrivateRegistryFeeds);
feedsToConsider.UnionWith(privateRegistryFeeds);
}

var feedsToUse = CheckNugetFeedResponsiveness
? feedsToConsider.Where(reachableFeeds.Contains)
: feedsToConsider;

return FeedsToRestoreArgument(feedsToUse);
return feedsToUse;
}

/// <summary>
/// Constructs the list of NuGet sources to use for dotnet restore.
/// (1) Use the feeds we get from `dotnet nuget list source`
/// (2) Use private registries, if they are configured
/// </summary>
/// <param name="path">Path to project/solution</param>
/// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
/// <returns>A string representing the NuGet sources argument for the restore command.</returns>
public string? MakeDotnetRestoreSourcesArgument(string path, ImmutableHashSet<string> reachableFeeds)
{
// Do not construct a set of explicit NuGet sources to use for restore.
if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
{
return null;
}

var feedsToUse = FeedsToUse(path, reachableFeeds);

return FeedsToRestoreArgument(feedsToUse, "-s");
}

private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
Expand Down Expand Up @@ -256,7 +328,7 @@ private HashSet<string> GetExcludedFeeds()
/// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured
/// to be excluded from the check) or false otherwise.
/// </returns>
public bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reachableFeeds)
private bool CheckSpecifiedFeeds(ImmutableHashSet<string> feeds, out ImmutableHashSet<string> reachableFeeds)
{
// Exclude any feeds from the feed check that are configured by the corresponding environment variable.
// These feeds are always assumed to be reachable.
Expand All @@ -272,10 +344,10 @@ public bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reach
return true;
}).ToHashSet();

reachableFeeds = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout).ToHashSet();
var reachable = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout);

// Always consider feeds excluded for the reachability check as reachable.
reachableFeeds.UnionWith(feeds.Where(feed => excludedFeeds.Contains(feed)));
reachableFeeds = reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();

return isTimeout;
}
Expand Down Expand Up @@ -327,7 +399,7 @@ private List<string> GetReachableNuGetFeeds(HashSet<string> feedsToCheck, bool i
return reachableFeeds;
}

public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
public List<string> GetReachableFallbackNugetFeeds(ImmutableHashSet<string>? feedsFromNugetConfigs)
{
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
if (fallbackFeeds.Count == 0)
Expand All @@ -350,7 +422,7 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _);
}

public (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
private ImmutableHashSet<string> GetExplicitFeeds()
{
var nugetConfigs = fileProvider.NugetConfigs;

Expand All @@ -372,14 +444,21 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
// in addition to the ones that are configured in `nuget.config` files.
if (HasPrivateRegistryFeeds)
{
logger.LogInfo($"Found {PrivateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", PrivateRegistryFeeds.OrderBy(f => f))}");
explicitFeeds.UnionWith(PrivateRegistryFeeds);
logger.LogInfo($"Found {privateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", privateRegistryFeeds.OrderBy(f => f))}");
explicitFeeds.UnionWith(privateRegistryFeeds);
}

return explicitFeeds.ToImmutableHashSet();
}

private ImmutableHashSet<string> GetAllFeeds()
{
var nugetConfigs = fileProvider.NugetConfigs;

HashSet<string> allFeeds = [];

// Add all explicitFeeds to the set of all feeds.
allFeeds.UnionWith(explicitFeeds);
allFeeds.UnionWith(ExplicitFeeds);

// Obtain the list of feeds from the root source directory.
// If a NuGet file is present it will be respected, otherwise we will just get the machine/environment specific feeds.
Expand All @@ -399,7 +478,7 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug

logger.LogInfo($"Found {allFeeds.Count} NuGet feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");

return (explicitFeeds, allFeeds);
return allFeeds.ToImmutableHashSet();
}

[GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
Expand Down
Loading
Loading