Skip to content

Commit 71acbd7

Browse files
danfiedler-msftdanfiedler-msft
authored
fix failing download test by using API for CPAN version info rather than parsing HTML (#511)
1 parent 7aebdd4 commit 71acbd7

8 files changed

Lines changed: 386 additions & 32 deletions

File tree

‎src/OSSGadget.sln‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Shared.CLI", "Shared.CLI\Sh
1818
EndProject
1919
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "oss-gadget-cli", "oss-gadget-cli\oss-gadget-cli.csproj", "{9B2B9345-186C-44B7-90C3-0D7A892D464F}"
2020
EndProject
21+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}"
22+
EndProject
23+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{F57911B9-C269-4238-BF89-5F62CD7C9F78}"
24+
EndProject
25+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared.Lib.Tests", "Shared.Lib.Tests\Shared.Lib.Tests.csproj", "{5BC950E8-32DB-4973-A5A6-E78739741D28}"
26+
EndProject
2127
Global
2228
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2329
Debug|Any CPU = Debug|Any CPU
@@ -43,10 +49,22 @@ Global
4349
{9B2B9345-186C-44B7-90C3-0D7A892D464F}.Debug|Any CPU.Build.0 = Debug|Any CPU
4450
{9B2B9345-186C-44B7-90C3-0D7A892D464F}.Release|Any CPU.ActiveCfg = Release|Any CPU
4551
{9B2B9345-186C-44B7-90C3-0D7A892D464F}.Release|Any CPU.Build.0 = Release|Any CPU
52+
{5BC950E8-32DB-4973-A5A6-E78739741D28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
53+
{5BC950E8-32DB-4973-A5A6-E78739741D28}.Debug|Any CPU.Build.0 = Debug|Any CPU
54+
{5BC950E8-32DB-4973-A5A6-E78739741D28}.Release|Any CPU.ActiveCfg = Release|Any CPU
55+
{5BC950E8-32DB-4973-A5A6-E78739741D28}.Release|Any CPU.Build.0 = Release|Any CPU
4656
EndGlobalSection
4757
GlobalSection(SolutionProperties) = preSolution
4858
HideSolutionNode = FALSE
4959
EndGlobalSection
60+
GlobalSection(NestedProjects) = preSolution
61+
{812DF4A5-057B-4227-A3E8-4DFF67DBE07A} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
62+
{182CA72B-6BB1-400E-83FC-B35558928BAB} = {F57911B9-C269-4238-BF89-5F62CD7C9F78}
63+
{0023A885-5255-46A0-BAF7-F419FFA00AE4} = {F57911B9-C269-4238-BF89-5F62CD7C9F78}
64+
{66CE54D2-40AA-41CB-A487-3FE44E38BFE0} = {F57911B9-C269-4238-BF89-5F62CD7C9F78}
65+
{9B2B9345-186C-44B7-90C3-0D7A892D464F} = {F57911B9-C269-4238-BF89-5F62CD7C9F78}
66+
{5BC950E8-32DB-4973-A5A6-E78739741D28} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
67+
EndGlobalSection
5068
GlobalSection(ExtensibilityGlobals) = postSolution
5169
SolutionGuid = {587898D9-D446-4D21-A442-C77D3477B2AF}
5270
EndGlobalSection

‎src/Shared.CLI/Helpers/PackageDownloader.cs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -289,8 +289,6 @@ public async Task<List<PackageURL>> GetPackageVersionsToProcess(PackageURL purl)
289289

290290
private string destinationDirectory { get; set; }
291291

292-
private bool usingTempDir;
293-
294292
// folders created
295293
private List<string> downloadPaths { get; set; } = new List<string>();
296294

‎src/Shared.Lib.Tests/Fakes/FakeHttpMessageHandler.cs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
2+
3+
namespace Microsoft.CST.OpenSource.Tests.Fakes;
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Net;
7+
using System.Threading.Tasks;
8+
9+
10+
// manual fake class necessary because NSubstitute can't mock the internal protected Send/SendAsync methods of HttpMessageHandler
11+
public class FakeHttpMessageHandler : HttpMessageHandler
12+
{
13+
private readonly Dictionary<string, HttpResponseMessage> _responseMap;
14+
public FakeHttpMessageHandler(Dictionary<string, HttpResponseMessage> responseMap)
15+
{
16+
_responseMap = responseMap;
17+
}
18+
19+
public Dictionary<string, int> PathsRequested { get; private set; } = [];
20+
21+
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
22+
{
23+
HttpResponseMessage response = Send(request, cancellationToken);
24+
return Task.FromResult(response);
25+
}
26+
27+
private readonly HttpResponseMessage _notFoundResponse = new()
28+
{
29+
StatusCode = HttpStatusCode.NotFound
30+
};
31+
32+
protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
33+
{
34+
string? url = request.RequestUri?.PathAndQuery;
35+
if (string.IsNullOrEmpty(url))
36+
{
37+
throw new Exception("Invalid test setup. Expected HttpRequestMessage.RequestUri to be non-null.");
38+
}
39+
40+
if (PathsRequested.TryGetValue(url, out _))
41+
{
42+
PathsRequested[url]++;
43+
}
44+
else
45+
{
46+
PathsRequested.Add(url, 1);
47+
}
48+
49+
if (_responseMap.TryGetValue(url, out HttpResponseMessage? response) && response != null)
50+
{
51+
return response;
52+
}
53+
54+
return _notFoundResponse;
55+
}
56+
}

‎src/Shared.Lib.Tests/PackageManagers/CPAN/CPANMetadataClientTests.cs‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
namespace Microsoft.CST.OpenSource.Tests.PackageManagers.CPAN;
2+
3+
using AutoFixture;
4+
using FluentAssertions;
5+
using Microsoft.CST.OpenSource.PackageManagers.CPAN;
6+
using Microsoft.CST.OpenSource.Tests.Fakes;
7+
using Microsoft.Extensions.Configuration;
8+
using NSubstitute;
9+
using System.Net.Http;
10+
using System.Threading.Tasks;
11+
12+
public class CPANMetadataClientTests
13+
{
14+
private readonly Fixture _autoFixture = new();
15+
16+
[Fact]
17+
public async Task When_api_returns_404_then_result_status_is_404()
18+
{
19+
string packageName = _autoFixture.Create<string>();
20+
string expectedUrl = $"/v1/release/_search?q=distribution:{packageName}&fields=name,version&size=100";
21+
Dictionary<string, HttpResponseMessage> httpResponseMap = new();
22+
FakeHttpMessageHandler httpMessageHandler = new(httpResponseMap);
23+
IHttpClientFactory httpClientFactory = SetupHttpClientFactory(httpMessageHandler);
24+
CPANMetadataClient metadataClient = new(httpClientFactory);
25+
26+
VersionsResult versionsResult = await metadataClient.GetPackageVersions(packageName, CancellationToken.None);
27+
28+
httpMessageHandler.PathsRequested.ContainsKey(expectedUrl).Should().BeTrue();
29+
httpMessageHandler.PathsRequested[expectedUrl].Should().Be(1);
30+
versionsResult.StatusCode.Should().Be(System.Net.HttpStatusCode.NotFound);
31+
versionsResult.ResponseContent.Should().Be(string.Empty);
32+
versionsResult.Versions.Should().BeEmpty();
33+
}
34+
35+
[Fact]
36+
public async Task When_api_returns_two_versions_then_result_has_two_expected_versions()
37+
{
38+
string packageName = _autoFixture.Create<string>();
39+
string expectedUrl = $"/v1/release/_search?q=distribution:{packageName}&fields=name,version&size=100";
40+
string apiResponseJson = @"{
41+
""_shards"" : {
42+
""failed"" : 0,
43+
""total"" : 3,
44+
""successful"" : 3
45+
},
46+
""took"" : 3,
47+
""hits"" : {
48+
""total"" : 4,
49+
""hits"" : [
50+
{
51+
""_id"" : ""fUw34Wazh12lcRrXHFoG8ZNW7Rs"",
52+
""_score"" : 14.512648,
53+
""_index"" : ""release_01"",
54+
""fields"" : {
55+
""version"" : ""v0.0.1"",
56+
""name"" : ""Data-Rand-v0.0.1""
57+
},
58+
""_type"" : ""release""
59+
},
60+
{
61+
""_score"" : 14.512648,
62+
""_index"" : ""release_01"",
63+
""fields"" : {
64+
""version"" : ""0.0.4"",
65+
""name"" : ""Data-Rand-0.0.4""
66+
},
67+
""_type"" : ""release"",
68+
""_id"" : ""f977gt_5D0ET48zZWEhBcFwhiyY""
69+
},
70+
],
71+
""max_score"" : 14.512648
72+
},
73+
""timed_out"" : false
74+
}";
75+
HttpResponseMessage apiResponse = new()
76+
{
77+
Content = new StringContent(apiResponseJson),
78+
StatusCode = System.Net.HttpStatusCode.OK
79+
};
80+
Dictionary<string, HttpResponseMessage> httpResponseMap = new();
81+
httpResponseMap.Add(expectedUrl, apiResponse);
82+
FakeHttpMessageHandler httpMessageHandler = new(httpResponseMap);
83+
IHttpClientFactory httpClientFactory = SetupHttpClientFactory(httpMessageHandler);
84+
CPANMetadataClient metadataClient = new(httpClientFactory);
85+
86+
VersionsResult versionsResult = await metadataClient.GetPackageVersions(packageName, CancellationToken.None);
87+
88+
httpMessageHandler.PathsRequested.ContainsKey(expectedUrl).Should().BeTrue();
89+
httpMessageHandler.PathsRequested[expectedUrl].Should().Be(1);
90+
versionsResult.StatusCode.Should().Be(System.Net.HttpStatusCode.OK);
91+
versionsResult.ResponseContent.Should().Be(apiResponseJson);
92+
versionsResult.Versions.Count().Should().Be(2);
93+
versionsResult.Versions[0].Should().Be("v0.0.1");
94+
versionsResult.Versions[1].Should().Be("0.0.4");
95+
}
96+
97+
private static IHttpClientFactory SetupHttpClientFactory(FakeHttpMessageHandler httpMessageHandler)
98+
{
99+
IHttpClientFactory httpClientFactory = Substitute.For<IHttpClientFactory>();
100+
httpClientFactory.CreateClient(Arg.Any<string>()).Returns(
101+
x => new HttpClient(httpMessageHandler, false)
102+
);
103+
return httpClientFactory;
104+
}
105+
}

‎src/Shared.Lib.Tests/Shared.Lib.Tests.csproj‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net8.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
<IsPackable>false</IsPackable>
8+
<IsTestProject>true</IsTestProject>
9+
<RootNamespace>Microsoft.CST.OpenSource.Tests</RootNamespace>
10+
<Description>OSS Gadget - Shared Library Functionality Tests</Description>
11+
<RepositoryType>GitHub</RepositoryType>
12+
<RepositoryUrl>https://github.com/Microsoft/OSSGadget</RepositoryUrl>
13+
<AssemblyName>OSSGadget.Shared.Lib.Tests</AssemblyName>
14+
<Nullable>enable</Nullable>
15+
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
16+
<Authors>Microsoft</Authors>
17+
<Company>Microsoft</Company>
18+
</PropertyGroup>
19+
20+
<ItemGroup>
21+
<PackageReference Include="AutoFixture" Version="4.18.1" />
22+
<PackageReference Include="AwesomeAssertions" Version="8.2.0" />
23+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
24+
<PackageReference Include="NSubstitute" Version="5.3.0" />
25+
<PackageReference Include="xunit" Version="2.9.3" />
26+
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.0">
27+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
28+
<PrivateAssets>all</PrivateAssets>
29+
</PackageReference>
30+
</ItemGroup>
31+
32+
<ItemGroup>
33+
<ProjectReference Include="..\Shared.CLI\Shared.CLI.csproj" />
34+
</ItemGroup>
35+
36+
<ItemGroup>
37+
<Using Include="Xunit" />
38+
</ItemGroup>
39+
40+
</Project>

‎src/Shared/PackageManagers/CPAN/CPANMetadataClient.cs‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
2+
3+
namespace Microsoft.CST.OpenSource.PackageManagers.CPAN;
4+
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Net;
8+
using System.Net.Http;
9+
using System.Text.Json;
10+
using System.Threading;
11+
using System.Threading.Tasks;
12+
13+
// todo: consider moving VersionsResult to a better namespace if re-used for other repositories
14+
public class VersionsResult
15+
{
16+
public bool PackageFound { get; set; }
17+
public List<string> Versions { get; set; } = new List<string>();
18+
public HttpStatusCode StatusCode { get; set; }
19+
public string ResponseContent { get; set; } = string.Empty;
20+
}
21+
22+
public interface ICPANMetadataClient
23+
{
24+
Task<VersionsResult> GetPackageVersions(string packageName, CancellationToken cancellationToken);
25+
}
26+
27+
public class CPANMetadataClient : ICPANMetadataClient
28+
{
29+
// metacpan-api docs: https://github.com/metacpan/metacpan-api/blob/master/docs/API-docs.md
30+
31+
private readonly IHttpClientFactory _httpClientFactory;
32+
private readonly Uri _metadataUri = new Uri("https://fastapi.metacpan.org/v1/");
33+
34+
private static readonly JsonSerializerOptions _jsonSerializerOptions = new JsonSerializerOptions
35+
{
36+
AllowTrailingCommas = true
37+
};
38+
39+
protected static readonly NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger();
40+
41+
public CPANMetadataClient(IHttpClientFactory httpClientFactory)
42+
{
43+
_httpClientFactory = httpClientFactory;
44+
45+
// TODO: consider making _metadataUri configurable via IConfiguration ala:
46+
//string? metadataEndpoint = config["cpan.urls.metadata"];
47+
//if(!string.IsNullOrEmpty(metadataEndpoint))
48+
//{
49+
// _metadataUri = new Uri(metadataEndpoint);
50+
//}
51+
}
52+
53+
public async Task<VersionsResult> GetPackageVersions(string packageName, CancellationToken cancellationToken)
54+
{
55+
VersionsResult result = new();
56+
57+
HttpClient httpClient = _httpClientFactory.CreateClient(GetType().Name);
58+
httpClient.BaseAddress = _metadataUri;
59+
60+
string urlEncodedName = WebUtility.UrlEncode(packageName);
61+
string urlPath = $"release/_search?q=distribution:{urlEncodedName}&fields=name,version&size=100";
62+
63+
HttpResponseMessage httpResponse = await httpClient.GetAsync(urlPath, cancellationToken);
64+
result.StatusCode = httpResponse.StatusCode;
65+
66+
if(httpResponse.IsSuccessStatusCode)
67+
{
68+
string json = await httpResponse.Content.ReadAsStringAsync();
69+
result.ResponseContent = json;
70+
71+
CPANVersionResponse? versionResponse = JsonSerializer.Deserialize<CPANVersionResponse>(json, _jsonSerializerOptions);
72+
if(versionResponse != null)
73+
{
74+
AddResponseVersionsToResult(result, versionResponse);
75+
}
76+
else
77+
{
78+
Logger.Warn("Failed to deserialize response from CPAN metadata API");
79+
}
80+
}
81+
82+
return result;
83+
}
84+
85+
private static void AddResponseVersionsToResult(VersionsResult result, CPANVersionResponse versionResponse)
86+
{
87+
if(versionResponse.Hits.Total > 0)
88+
{
89+
result.PackageFound = true;
90+
91+
foreach (CPANVersionHit hit in versionResponse.Hits.Hits)
92+
{
93+
result.Versions.Add(hit.Fields.Version);
94+
}
95+
}
96+
}
97+
}

‎src/Shared/PackageManagers/CPAN/CPANVersionResponse.cs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
2+
3+
namespace Microsoft.CST.OpenSource.PackageManagers.CPAN;
4+
5+
using System.Collections.Generic;
6+
using System.Text.Json.Serialization;
7+
8+
public class CPANVersionResponse
9+
{
10+
[JsonPropertyName("took")]
11+
public int Took { get; set; }
12+
13+
[JsonPropertyName("hits")]
14+
public CPANVersionHitCollection Hits { get; set; } = new();
15+
16+
[JsonPropertyName("timed_out")]
17+
public bool TimedOut { get; set; }
18+
}
19+
20+
public class CPANVersionHitCollection
21+
{
22+
[JsonPropertyName("total")]
23+
public int Total { get; set; }
24+
25+
[JsonPropertyName("hits")]
26+
public List<CPANVersionHit> Hits { get; set; } = new List<CPANVersionHit>();
27+
}
28+
29+
public class CPANVersionHit
30+
{
31+
[JsonPropertyName("_id")]
32+
public string Id { get; set; } = string.Empty;
33+
34+
[JsonPropertyName("_index")]
35+
public string Index { get; set; } = string.Empty;
36+
37+
[JsonPropertyName("fields")]
38+
public CPANVersionFields Fields { get; set; } = new();
39+
40+
[JsonPropertyName("_type")]
41+
public string Type { get; set; } = string.Empty;
42+
}
43+
44+
public class CPANVersionFields
45+
{
46+
[JsonPropertyName("version")]
47+
public string Version { get; set; } = string.Empty;
48+
49+
[JsonPropertyName("name")]
50+
public string Name { get; set; } = string.Empty;
51+
}

0 commit comments

Comments
 (0)