From b5bae1b48820e5336dba0205dd446e8b3dc0a0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Wed, 20 Nov 2024 10:16:35 +0100 Subject: [PATCH] Split GivenThatWeWantToRunILLink test class into multiple so Helix can run them separately Fixes https://github.com/dotnet/sdk/issues/44895 --- .../GivenThatWeWantToRunILLink.cs | 266 ++++++++++-------- 1 file changed, 141 insertions(+), 125 deletions(-) diff --git a/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs b/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs index 435c48fb5835..03e3171e3c00 100644 --- a/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs +++ b/test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs @@ -11,12 +11,14 @@ using Microsoft.NET.Build.Tasks; using Newtonsoft.Json.Linq; using static Microsoft.NET.Publish.Tests.PublishTestUtils; +using static Microsoft.NET.Publish.Tests.ILLinkTestUtils; namespace Microsoft.NET.Publish.Tests { - public class GivenThatWeWantToRunILLink : SdkTest + // this test class is split up arbitrarily so Helix can run tests in multiple workitems + public class GivenThatWeWantToRunILLink1 : SdkTest { - public GivenThatWeWantToRunILLink(ITestOutputHelper log) : base(log) + public GivenThatWeWantToRunILLink1(ITestOutputHelper log) : base(log) { } @@ -28,7 +30,7 @@ public void ILLink_only_runs_when_switch_is_enabled(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -42,7 +44,7 @@ public void ILLink_only_runs_when_switch_is_enabled(string targetFramework) var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); // Linker inputs are kept, including unused assemblies File.Exists(publishedDll).Should().BeTrue(); @@ -51,7 +53,7 @@ public void ILLink_only_runs_when_switch_is_enabled(string targetFramework) var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeTrue(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeTrue(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeTrue(); } [RequiresMSBuildVersionTheory("17.0.0.32901")] @@ -65,7 +67,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceClassLibAsPackage); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceClassLibAsPackage); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework + referenceClassLibAsPackage) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -81,7 +83,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe var linkedDll = Path.Combine(linkedDirectory, $"{projectName}.dll"); var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); // Intermediate assembly is kept by linker and published, but not unused assemblies File.Exists(linkedDll).Should().BeTrue(); @@ -92,7 +94,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, projectName).Should().BeTrue(); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeFalse(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeFalse(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeFalse(); } [RequiresMSBuildVersionTheory("17.0.0.32901")] @@ -104,7 +106,7 @@ public void ILLink_links_simple_app_without_analysis_warnings_and_it_runs(string var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.AdditionalProperties["PublishTrimmed"] = "true"; var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework + trimMode); @@ -132,7 +134,7 @@ public void PublishTrimmed_fails_when_no_matching_pack_is_found(string targetFra var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => { @@ -161,7 +163,7 @@ public void PublishTrimmed_fails_for_unsupported_target_framework(string targetF var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.AdditionalProperties["PublishTrimmed"] = "true"; testProject.AdditionalProperties["NoWarn"] = "NETSDK1138"; // Silence warning about targeting EOL TFMs var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); @@ -194,7 +196,7 @@ public void IsTrimmable_warns_when_expected_for_not_correctly_multitargeted_libr var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFrameworks); - var testProject = CreateTestProjectForILLinkTesting(targetFrameworks, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFrameworks, projectName); testProject.AdditionalProperties["IsTrimmable"] = "true"; testProject.AdditionalProperties["NoWarn"] = "NETSDK1138"; // Silence warning about targeting EOL TFMs var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFrameworks) @@ -221,7 +223,7 @@ public void RequiresILLinkPack_errors_for_unsupported_target_framework(string ta var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.AdditionalProperties["_RequiresILLinkPack"] = "true"; var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); @@ -255,7 +257,7 @@ public void PrepareForILLink_can_set_IsTrimmable(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => SetMetadata(project, referenceProjectName, "IsTrimmable", "True")); @@ -280,7 +282,7 @@ public void PrepareForILLink_can_set_TrimMode(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => SetMetadata(project, referenceProjectName, "TrimMode", "link")); @@ -310,7 +312,7 @@ public void ILLink_respects_global_TrimMode(string targetFramework, string trimM var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework + trimMode) .WithProjectChanges(project => SetGlobalTrimMode(project, trimMode)) .WithProjectChanges(project => SetMetadata(project, referenceProjectName, "IsTrimmable", "True")) @@ -346,7 +348,7 @@ public void ILLink_roots_IntermediateAssembly(string targetFramework) var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => SetGlobalTrimMode(project, "link")) .WithProjectChanges(project => SetMetadata(project, projectName, "IsTrimmable", "True")); @@ -371,7 +373,7 @@ public void ILLink_respects_TrimmableAssembly(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); testProject.AddItem("TrimmableAssembly", "Include", referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); @@ -748,7 +750,7 @@ public void ILLink_errors_fail_the_build(string targetFramework) var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); // Set up a project with an invalid feature substitution, just to produce an error. - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); testProject.SourceFiles[$"{projectName}.xml"] = $@" @@ -775,6 +777,14 @@ public void ILLink_errors_fail_the_build(string targetFramework) File.Exists(linkSemaphore).Should().BeFalse(); File.Exists(publishedDll).Should().BeFalse(); } + } + + // this test class is split up arbitrarily so Helix can run tests in multiple workitems + public class GivenThatWeWantToRunILLink2 : SdkTest + { + public GivenThatWeWantToRunILLink2(ITestOutputHelper log) : base(log) + { + } [RequiresMSBuildVersionTheory("17.0.0.32901")] [MemberData(nameof(Net6Plus), MemberType = typeof(PublishTestUtils))] @@ -826,7 +836,7 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_copyused(s } } - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -917,7 +927,7 @@ public void ILLink_verify_analysis_warnings_framework_assemblies(string targetFr }); } - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -936,7 +946,7 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_link(strin var projectName = "AnalysisWarningsOnHelloWorldApp"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -952,7 +962,7 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_link_5_0() var projectName = "AnalysisWarningsOnHelloWorldApp"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name)); @@ -962,61 +972,6 @@ public void ILLink_verify_analysis_warnings_hello_world_app_trim_mode_link_5_0() ValidateWarningsOnHelloWorldApp(publishCommand, result, new List(), targetFramework, rid); } - private void ValidateWarningsOnHelloWorldApp(PublishCommand publishCommand, CommandResult result, List expectedWarnings, string targetFramework, string rid, bool useRegex = false) - { - // This checks that there are no unexpected warnings, but does not cause failures for missing expected warnings. - var warnings = result.StdOut.Split('\n', '\r').Where(line => line.Contains("warning IL")); - - // This should also detect unexpected duplicates of expected warnings. - // Each expected warning string/regex matches at most one warning. - List extraWarnings = new(); - foreach (var warning in warnings) - { - bool expected = false; - for (int i = 0; i < expectedWarnings.Count; i++) - { - if ((useRegex && Regex.IsMatch(warning, expectedWarnings[i])) || - (!useRegex && warning.Contains(expectedWarnings[i]))) - { - expectedWarnings.RemoveAt(i); - expected = true; - break; - } - } - - if (!expected) - { - extraWarnings.Add(warning); - } - } - - StringBuilder errorMessage = new(); - - if (extraWarnings.Any()) - { - // Print additional information to recognize which framework assemblies are being used. - errorMessage.AppendLine($"Target framework from test: {targetFramework}"); - errorMessage.AppendLine($"Runtime identifier: {rid}"); - - // Get the array of runtime assemblies inside the publish folder. - string[] runtimeAssemblies = Directory.GetFiles(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "*.dll"); - var paths = new List(runtimeAssemblies); - var resolver = new PathAssemblyResolver(paths); - var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); - using (mlc) - { - Assembly assembly = mlc.LoadFromAssemblyPath(Path.Combine(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "System.Private.CoreLib.dll")); - string assemblyVersionInfo = (string)assembly.CustomAttributes.Where(ca => ca.AttributeType.Name == "AssemblyInformationalVersionAttribute").Select(ca => ca.ConstructorArguments[0].Value).FirstOrDefault(); - errorMessage.AppendLine($"Runtime Assembly Informational Version: {assemblyVersionInfo}"); - } - errorMessage.AppendLine($"The execution of a hello world app generated a diff in the number of warnings the app produces{Environment.NewLine}"); - errorMessage.AppendLine("Test output contained the following extra linker warnings:"); - foreach (var extraWarning in extraWarnings) - errorMessage.AppendLine($"+ {extraWarning}"); - } - Assert.True(!extraWarnings.Any(), errorMessage.ToString()); - } - [RequiresMSBuildVersionTheory("17.0.0.32901")] [MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))] public void TrimmingOptions_are_defaulted_correctly_on_trimmed_apps(string targetFramework) @@ -1024,7 +979,7 @@ public void TrimmingOptions_are_defaulted_correctly_on_trimmed_apps(string targe var projectName = "HelloWorld"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: projectName + targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1097,7 +1052,7 @@ public void ILLink_accepts_root_descriptor(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)) .WithProjectChanges(project => AddRootDescriptor(project, $"{referenceProjectName}.xml")); @@ -1137,7 +1092,7 @@ public void ILLink_error_on_nonboolean_optimization_flag(string property) var targetFramework = "net5.0"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: property); var publishCommand = new PublishCommand(testAsset); @@ -1153,7 +1108,7 @@ public void ILLink_respects_feature_settings_from_host_config() var targetFramework = "net5.0"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, // Reference the classlib to ensure its XML is processed. addAssemblyReference: true, // Set up a conditional feature substitution for the "FeatureDisabled" property @@ -1186,7 +1141,7 @@ public void ILLink_ignores_host_config_settings_with_link_false() var targetFramework = "net5.0"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, // Reference the classlib to ensure its XML is processed. addAssemblyReference: true, // Set up a conditional feature substitution for the "FeatureDisabled" property @@ -1219,7 +1174,7 @@ public void ILLink_runs_incrementally(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1250,7 +1205,7 @@ public void ILLink_old_defaults_keep_nonframework(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1265,7 +1220,7 @@ public void ILLink_old_defaults_keep_nonframework(string targetFramework) var linkedDll = Path.Combine(linkedDirectory, $"{projectName}.dll"); var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); File.Exists(linkedDll).Should().BeTrue(); File.Exists(publishedDll).Should().BeTrue(); @@ -1275,7 +1230,7 @@ public void ILLink_old_defaults_keep_nonframework(string targetFramework) var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, projectName).Should().BeTrue(); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeTrue(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeFalse(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeFalse(); } [RequiresMSBuildVersionFact("17.0.0.32901")] @@ -1286,7 +1241,7 @@ public void ILLink_net7_defaults_trim_nonframework() var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1301,7 +1256,7 @@ public void ILLink_net7_defaults_trim_nonframework() var linkedDll = Path.Combine(linkedDirectory, $"{projectName}.dll"); var publishedDll = Path.Combine(publishDirectory, $"{projectName}.dll"); var unusedDll = Path.Combine(publishDirectory, $"{referenceProjectName}.dll"); - var unusedFrameworkDll = Path.Combine(publishDirectory, $"{unusedFrameworkAssembly}.dll"); + var unusedFrameworkDll = Path.Combine(publishDirectory, $"{UnusedFrameworkAssembly}.dll"); File.Exists(linkedDll).Should().BeTrue(); File.Exists(publishedDll).Should().BeTrue(); @@ -1311,7 +1266,7 @@ public void ILLink_net7_defaults_trim_nonframework() var depsFile = Path.Combine(publishDirectory, $"{projectName}.deps.json"); DoesDepsFileHaveAssembly(depsFile, projectName).Should().BeTrue(); DoesDepsFileHaveAssembly(depsFile, referenceProjectName).Should().BeFalse(); - DoesDepsFileHaveAssembly(depsFile, unusedFrameworkAssembly).Should().BeFalse(); + DoesDepsFileHaveAssembly(depsFile, UnusedFrameworkAssembly).Should().BeFalse(); } [RequiresMSBuildVersionTheory("17.0.0.32901")] @@ -1322,7 +1277,7 @@ public void ILLink_does_not_include_leftover_artifacts_on_second_run(string targ var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, referenceProjectIdentifier: targetFramework); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)) .WithProjectChanges(project => AddRootDescriptor(project, $"{referenceProjectName}.xml")); @@ -1375,7 +1330,7 @@ public void ILLink_keeps_symbols_by_default(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1408,7 +1363,7 @@ public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targ var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1427,6 +1382,14 @@ public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targ File.Exists(linkedPdb).Should().BeFalse(); File.Exists(publishedPdb).Should().BeFalse(); } + } + + // this test class is split up arbitrarily so Helix can run tests in multiple workitems + public class GivenThatWeWantToRunILLink3 : SdkTest + { + public GivenThatWeWantToRunILLink3(ITestOutputHelper log) : base(log) + { + } [RequiresMSBuildVersionTheory("17.0.0.32901")] [MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))] @@ -1436,7 +1399,7 @@ public void ILLink_accepts_option_to_remove_symbols(string targetFramework) var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1464,7 +1427,7 @@ public void ILLink_symbols_option_can_override_defaults_from_debugger_support(st var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework) .WithProjectChanges(project => EnableNonFrameworkTrimming(project)); @@ -1616,7 +1579,7 @@ public void ILLink_error_on_portable_app(string targetFramework) var projectName = "HelloWorld"; var referenceProjectName = "ClassLibForILLink"; - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName, setSelfContained: false); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName, setSelfContained: false); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1634,7 +1597,7 @@ public void ILLink_displays_informational_warning_up_to_net5_by_default(string t var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1651,7 +1614,7 @@ public void ILLink_displays_informational_warning_when_trim_analysis_warnings_ar var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1669,7 +1632,7 @@ public void ILLink_dont_display_informational_warning_by_default_on_net6plus(str var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1687,7 +1650,7 @@ public void ILLink_dont_display_time_awareness_message_on_incremental_build(stri var referenceProjectName = "ClassLibForILLink"; var rid = EnvironmentInfo.GetCompatibleRid(targetFramework); - var testProject = CreateTestProjectForILLinkTesting(targetFramework, projectName, referenceProjectName); + var testProject = CreateTestProjectForILLinkTesting(_testAssetsManager, targetFramework, projectName, referenceProjectName); var testAsset = _testAssetsManager.CreateTestProject(testProject, identifier: targetFramework); var publishCommand = new PublishCommand(testAsset); @@ -1811,8 +1774,13 @@ public void Build_respects_PublishTrimmed_property(string targetFramework) // just setting PublishTrimmed doesn't inject the IsTrimmable attribute AssemblyInfo.Get(assemblyPath).ContainsKey("AssemblyMetadataAttribute").Should().BeFalse(); } + } + + internal static class ILLinkTestUtils + { + public static string UnusedFrameworkAssembly = "System.IO"; - private static bool DoesImageHaveMethod(string path, string methodNameToCheck) + public static bool DoesImageHaveMethod(string path, string methodNameToCheck) { using (FileStream fs = new(path, FileMode.Open, FileAccess.Read)) using (var peReader = new PEReader(fs)) @@ -1829,7 +1797,7 @@ private static bool DoesImageHaveMethod(string path, string methodNameToCheck) return false; } - private static bool DoesDepsFileHaveAssembly(string depsFilePath, string assemblyName) + public static bool DoesDepsFileHaveAssembly(string depsFilePath, string assemblyName) { DependencyContext dependencyContext; using (var fs = File.OpenRead(depsFilePath)) @@ -1843,15 +1811,7 @@ private static bool DoesDepsFileHaveAssembly(string depsFilePath, string assembl Path.GetFileName(f) == $"{assemblyName}.dll"))); } - static string unusedFrameworkAssembly = "System.IO"; - - private TestAsset GetProjectReference(TestProject project, string callingMethod, string identifier) - { - var asset = _testAssetsManager.CreateTestProject(project, callingMethod: callingMethod, identifier: identifier); - return asset; - } - - private void AddRootDescriptor(XDocument project, string rootDescriptorFileName) + public static void AddRootDescriptor(XDocument project, string rootDescriptorFileName) { var ns = project.Root.Name.Namespace; @@ -1861,7 +1821,7 @@ private void AddRootDescriptor(XDocument project, string rootDescriptorFileName) new XAttribute("Include", rootDescriptorFileName))); } - private void RemoveRootDescriptor(XDocument project) + public static void RemoveRootDescriptor(XDocument project) { var ns = project.Root.Name.Namespace; @@ -1870,7 +1830,7 @@ private void RemoveRootDescriptor(XDocument project) .First().Remove(); } - private void SetMetadata(XDocument project, string assemblyName, string key, string value) + public static void SetMetadata(XDocument project, string assemblyName, string key, string value) { var ns = project.Root.Name.Namespace; var targetName = "SetTrimmerMetadata"; @@ -1892,7 +1852,7 @@ private void SetMetadata(XDocument project, string assemblyName, string key, str new XAttribute(key, value)))); } - private void SetGlobalTrimMode(XDocument project, string trimMode) + public static void SetGlobalTrimMode(XDocument project, string trimMode) { var ns = project.Root.Name.Namespace; @@ -1902,7 +1862,7 @@ private void SetGlobalTrimMode(XDocument project, string trimMode) trimMode)); } - private void SetTrimmerDefaultAction(XDocument project, string action) + public static void SetTrimmerDefaultAction(XDocument project, string action) { var ns = project.Root.Name.Namespace; @@ -1911,7 +1871,7 @@ private void SetTrimmerDefaultAction(XDocument project, string action) properties.Add(new XElement(ns + "TrimmerDefaultAction", action)); } - private void EnableNonFrameworkTrimming(XDocument project) + public static void EnableNonFrameworkTrimming(XDocument project) { // Used to override the default linker options for testing // purposes. The default roots non-framework assemblies, @@ -1934,10 +1894,10 @@ private void EnableNonFrameworkTrimming(XDocument project) new XAttribute("Include", "@(IntermediateAssembly->'%(FullPath)')"))); } - static readonly string substitutionsFilename = "ILLink.Substitutions.xml"; - - private void AddFeatureDefinition(TestProject testProject, string assemblyName) + public static void AddFeatureDefinition(TestProject testProject, string assemblyName) { + const string substitutionsFilename = "ILLink.Substitutions.xml"; + // Add a feature definition that replaces the FeatureDisabled property when DisableFeature is true. testProject.EmbeddedResources[substitutionsFilename] = $@" @@ -1956,7 +1916,7 @@ private void AddFeatureDefinition(TestProject testProject, string assemblyName) }); } - private void AddRuntimeConfigOption(XDocument project, bool trim) + public static void AddRuntimeConfigOption(XDocument project, bool trim) { var ns = project.Root.Name.Namespace; @@ -1967,7 +1927,7 @@ private void AddRuntimeConfigOption(XDocument project, bool trim) new XAttribute("Trim", trim.ToString())))); } - private TestProject CreateTestProjectWithAnalysisWarnings(string targetFramework, string projectName, bool isExe = true) + public static TestProject CreateTestProjectWithAnalysisWarnings(string targetFramework, string projectName, bool isExe = true) { var testProject = new TestProject() { @@ -2041,7 +2001,7 @@ public override void IL_2046() {} return testProject; } - private TestProject CreateTestProjectWithIsTrimmableAttributes( + public static TestProject CreateTestProjectWithIsTrimmableAttributes( string targetFramework, string projectName) { @@ -2146,7 +2106,8 @@ public static void UnusedMethod() return testProject; } - private TestProject CreateTestProjectForILLinkTesting( + public static TestProject CreateTestProjectForILLinkTesting( + TestAssetsManager testAssetsManager, string targetFrameworks, string mainProjectName, string referenceProjectName = null, @@ -2251,7 +2212,7 @@ public static void FeatureImplementation() if (usePackageReference) { - var referenceAsset = GetProjectReference(referenceProject, callingMethod, referenceProjectIdentifier ?? targetFrameworks); + var referenceAsset = testAssetsManager.CreateTestProject(referenceProject, callingMethod, referenceProjectIdentifier ?? targetFrameworks); testProject.ReferencedProjects.Add(referenceAsset.TestProject); } else @@ -2274,7 +2235,7 @@ public static void FeatureImplementation() return testProject; } - private void CheckILLinkVersion(TestAsset testAsset, string targetFramework) + public static void CheckILLinkVersion(TestAsset testAsset, string targetFramework) { var getKnownPacks = new GetValuesCommand(testAsset, "KnownILLinkPack", GetValuesCommand.ValueType.Item, targetFramework) { @@ -2294,5 +2255,60 @@ private void CheckILLinkVersion(TestAsset testAsset, string targetFramework) var illinkVersion = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(illinkTargetsPath))); illinkVersion.Should().Be(expectedVersion); } + + public static void ValidateWarningsOnHelloWorldApp(PublishCommand publishCommand, CommandResult result, List expectedWarnings, string targetFramework, string rid, bool useRegex = false) + { + // This checks that there are no unexpected warnings, but does not cause failures for missing expected warnings. + var warnings = result.StdOut.Split('\n', '\r').Where(line => line.Contains("warning IL")); + + // This should also detect unexpected duplicates of expected warnings. + // Each expected warning string/regex matches at most one warning. + List extraWarnings = new(); + foreach (var warning in warnings) + { + bool expected = false; + for (int i = 0; i < expectedWarnings.Count; i++) + { + if ((useRegex && Regex.IsMatch(warning, expectedWarnings[i])) || + (!useRegex && warning.Contains(expectedWarnings[i]))) + { + expectedWarnings.RemoveAt(i); + expected = true; + break; + } + } + + if (!expected) + { + extraWarnings.Add(warning); + } + } + + StringBuilder errorMessage = new(); + + if (extraWarnings.Any()) + { + // Print additional information to recognize which framework assemblies are being used. + errorMessage.AppendLine($"Target framework from test: {targetFramework}"); + errorMessage.AppendLine($"Runtime identifier: {rid}"); + + // Get the array of runtime assemblies inside the publish folder. + string[] runtimeAssemblies = Directory.GetFiles(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "*.dll"); + var paths = new List(runtimeAssemblies); + var resolver = new PathAssemblyResolver(paths); + var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); + using (mlc) + { + Assembly assembly = mlc.LoadFromAssemblyPath(Path.Combine(publishCommand.GetOutputDirectory(targetFramework: targetFramework, runtimeIdentifier: rid).FullName, "System.Private.CoreLib.dll")); + string assemblyVersionInfo = (string)assembly.CustomAttributes.Where(ca => ca.AttributeType.Name == "AssemblyInformationalVersionAttribute").Select(ca => ca.ConstructorArguments[0].Value).FirstOrDefault(); + errorMessage.AppendLine($"Runtime Assembly Informational Version: {assemblyVersionInfo}"); + } + errorMessage.AppendLine($"The execution of a hello world app generated a diff in the number of warnings the app produces{Environment.NewLine}"); + errorMessage.AppendLine("Test output contained the following extra linker warnings:"); + foreach (var extraWarning in extraWarnings) + errorMessage.AppendLine($"+ {extraWarning}"); + } + Assert.True(!extraWarnings.Any(), errorMessage.ToString()); + } } }