Skip to content
Open
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Immutable;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography.X509Certificates;
Expand All @@ -14,13 +15,51 @@ public class DependabotProxy : IDependabotProxy
/// <summary>
/// Represents configurations for package registries.
/// </summary>
/// <param name="Type">The type of package registry.</param>
/// <param name="URL">The URL of the package registry.</param>
public record class RegistryConfig(string Type, string URL);
public class RegistryConfig
{
/// <summary>
/// The type of the package registry.
/// </summary>
public string? Type { get; init; }

/// <summary>
/// The URL of the package registry.
/// </summary>
public string? Url { get; init; }

/// <summary>
/// A boolean indicating whether this registry replaces the base registry.
/// </summary>
[JsonProperty("replaces-base")]
public bool ReplacesBase { get; init; } = false;
};

public string Address { get; }

public HashSet<string> RegistryURLs { get; } = [];
/// <summary>
/// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry.
/// </summary>
private readonly Dictionary<string, bool> registryMapping = [];

private ImmutableHashSet<string>? registryURLs;
/// <summary>
/// Gets the set of registry URLs that have been configured as part of the organization-level
/// private registry configuration. This includes all registries, regardless of whether they replace
/// the default feeds.
/// </summary>
public ImmutableHashSet<string> RegistryURLs =>
registryURLs ??= registryMapping.Keys.ToImmutableHashSet();

private ImmutableHashSet<string>? registryBaseURLs;
/// <summary>
/// Gets the set of registry URLs that have been configured as part of the organization-level
/// private registry configuration and that replace the default registry. This is a subset of
/// <see cref="RegistryURLs"/>.
/// If non-empty, the set should be used as a replacement for the default registry during
/// package resolution.
/// </summary>
public ImmutableHashSet<string> RegistryBaseURLs =>
registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet();

public string? CertificatePath { get; private set; }

Expand Down Expand Up @@ -56,16 +95,22 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te
{
foreach (RegistryConfig registry in array)
{
if (string.IsNullOrWhiteSpace(registry.Url))
{
logger.LogDebug("Ignoring registry with empty URL.");
continue;
}

// The array contains all configured private registries, not just ones for C#.
// We ignore the non-C# ones here.
if (!registry.Type.Equals("nuget_feed"))
if (registry.Type is null || !registry.Type.Equals("nuget_feed"))
{
logger.LogDebug($"Ignoring registry at '{registry.URL}' since it is not of type 'nuget_feed'.");
logger.LogDebug($"Ignoring registry at '{registry.Url}' since it is not of type 'nuget_feed'.");
continue;
}

logger.LogInfo($"Found private registry at '{registry.URL}'");
RegistryURLs.Add(registry.URL);
logger.LogInfo($"Found private registry at '{registry.Url}'");
registryMapping.AddOrUpdateToLatest(registry.Url, registry.ReplacesBase);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ internal static class EnvironmentVariableNames

/// <summary>
/// Specifies the NuGet feeds to use for fallback NuGet dependency fetching. The value is a space-separated list of feed URLs.
/// The default value is `https://api.nuget.org/v3/index.json`.
/// </summary>
public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal sealed partial class FeedManager : IDisposable
{
internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json";
private const string PublicNugetOrg = "nuget.org";
private const string PublicDotNugetOrg = $".{PublicNugetOrg}";
internal const string PublicApiNugetOrgFeed = $"https://api{PublicDotNugetOrg}/v3/index.json";

private readonly ILogger logger;
private readonly IDotNet dotnet;
private readonly IFileProvider fileProvider;
private readonly DependencyDirectory emptyPackageDirectory;
private readonly ImmutableHashSet<string> privateRegistryFeeds;
private readonly bool hasPrivateRegistryBaseFeeds;
private readonly ImmutableHashSet<string> privateRegistryBaseFeeds;
private readonly IFeedManagerIO feedManagerIo;

/// <summary>
Expand Down Expand Up @@ -72,14 +76,33 @@ internal sealed partial class FeedManager : IDisposable
/// </summary>
public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;

private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds;

/// <summary>
/// Gets the list of default NuGet feeds that are configured in the environment.
/// This is either the public NuGet feed or a set of feeds specified by the environment.
/// </summary>
public ImmutableHashSet<string> DefaultFeeds { get; init; }

/// <summary>
/// Gets the list of reachable default NuGet feeds.
/// </summary>
public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value;

public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
{
this.logger = logger;
this.dotnet = dotnet;
this.fileProvider = fileProvider;
this.feedManagerIo = feedManagerIo;
privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? [];
HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
privateRegistryBaseFeeds = dependabotProxy?.RegistryBaseURLs ?? [];
hasPrivateRegistryBaseFeeds = privateRegistryBaseFeeds.Count > 0;

DefaultFeeds = hasPrivateRegistryBaseFeeds
? privateRegistryBaseFeeds
: [PublicApiNugetOrgFeed];
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);

lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
Expand All @@ -96,13 +119,28 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
return reachableFallbackFeeds.ToImmutableHashSet();
});
lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(DefaultFeeds));
}

public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
: this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy))
{
}

private bool IsNugetOrgFeed(string url)
{
try
{
var uri = new Uri(url);
return uri.Host.EndsWith(PublicDotNugetOrg, StringComparison.InvariantCultureIgnoreCase) ||
string.Equals(uri.Host, PublicNugetOrg, StringComparison.InvariantCultureIgnoreCase);
}
catch (UriFormatException)
{
return false;
}
}

private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
{
var results = getNugetFeeds();
Expand All @@ -124,10 +162,18 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
continue;
}

if (!string.IsNullOrWhiteSpace(url))
if (hasPrivateRegistryBaseFeeds && IsNugetOrgFeed(url))
{
yield return url;
// Use private registry base feeds.
foreach (var feed in privateRegistryBaseFeeds)
{
logger.LogDebug($"Using private registry base feed '{feed}'.");
yield return feed;
}
continue;
}

yield return url;
}
}

Expand Down Expand Up @@ -266,22 +312,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe
return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
}

/// <summary>
/// Return true if the default NuGet feed is reachable, false otherwise.
/// If the reachability check is disabled, this method will always return true.
/// </summary>
/// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
public bool IsDefaultFeedReachable()
{
if (CheckNugetFeedResponsiveness)
{
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
}

return true;
}

/// <summary>
/// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
/// </summary>
Expand Down Expand Up @@ -315,8 +345,8 @@ private List<string> GetReachableFallbackNugetFeeds()
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
if (fallbackFeeds.Count == 0)
Comment on lines 345 to 346

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it's possible to set the environment variable for the fallback feeds in Default Setup, but I would still treat both those and the ones from the private registry configurations as explicit, user-provided configuration. Therefore, it might make sense to compute fallbackFeeds.UnionWith(defaultFeeds); even if there are fallback feeds configured in the environment variable. Alternatively, it might be good to at least visibly log that we are ignoring anything that doesn't come from the environment variable, when it is configured.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that the setting fallback nuget feeds using the environment variable was introduced as an override mechanism to force the use of the a specific set of feeds (if they are considered reachable) - I don't think we should do substitutions or additions in this case (or at least not as a part of this PR).

{
fallbackFeeds.Add(PublicNugetOrgFeed);
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
fallbackFeeds.UnionWith(DefaultFeeds);
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", DefaultFeeds.OrderBy(f => f))}");
Comment thread
michaelnebel marked this conversation as resolved.
Comment thread
michaelnebel marked this conversation as resolved.

var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Security.Cryptography.X509Certificates;

namespace Semmle.Extraction.CSharp.DependencyFetching
Expand All @@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable
/// <summary>
/// The URLs of package registries that are configured for the proxy.
/// </summary>
HashSet<string> RegistryURLs { get; }
ImmutableHashSet<string> RegistryURLs { get; }

/// <summary>
/// The URLs of package registries that replace the base registry.
/// </summary>
ImmutableHashSet<string> RegistryBaseURLs { get; }

/// <summary>
/// The path to the temporary file where the certificate is stored.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ private bool TryRestorePackageManually(string package, List<string> nugetSources
return true;
}

if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources.Count > 0)
if (!feedManager.CheckNugetFeedResponsiveness && !feedManager.HasPrivateRegistryFeeds && res.HasNugetPackageSourceError && nugetSources.Count > 0)
{
logger.LogDebug($"Trying to restore '{package}' without explicitly providing NuGet sources.");
// Restore could not be completed because the listed source is unavailable. Try without an explicit restore source argument.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore

private bool IsWindows => SystemBuildActions.Instance.IsWindows();

private bool? isDefaultFeedReachable;
private bool IsDefaultFeedReachable =>
isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable();

/// <summary>
/// Create the package manager for a specified source tree.
/// </summary>
Expand Down Expand Up @@ -169,15 +165,18 @@ private bool TryRestoreNugetPackage(string packagesConfig)

List<string> sourcesArgument = [];
var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList();
var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable;
var defaultFeeds = feedManager.CheckNugetFeedResponsiveness
? feedManager.ReachableDefaultFeeds
: feedManager.DefaultFeeds;
var useDefaultFeeds = feedsToUse.Count == 0 && defaultFeeds.Count > 0;

// Explicitly construct the sources to be used for the restore command when checking feed
// responsiveness, using private registries, or falling back to nuget.org.
if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed)
// responsiveness, using private registries, or falling back to default feeds.
if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds)
{
if (useDefaultFeed)
if (useDefaultFeeds)
{
feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
feedsToUse.AddRange(defaultFeeds);
}
var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
sourcesArgument = restoreFeeds.SelectMany<string, string>(feed => ["-Source", feed]).ToList();
Expand Down
Loading
Loading