diff --git a/src/Platform/Microsoft.Testing.Extensions.MSBuild/TestingPlatformBuilderHook.cs b/src/Platform/Microsoft.Testing.Extensions.MSBuild/TestingPlatformBuilderHook.cs index 4391827c3c..c2eb503d86 100644 --- a/src/Platform/Microsoft.Testing.Extensions.MSBuild/TestingPlatformBuilderHook.cs +++ b/src/Platform/Microsoft.Testing.Extensions.MSBuild/TestingPlatformBuilderHook.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.ComponentModel; + using Microsoft.Testing.Extensions.MSBuild; using Microsoft.Testing.Platform.Builder; @@ -35,10 +37,9 @@ public static void AddExtensions(ITestApplicationBuilder testApplicationBuilder, return; } - // The full-trust controller owns the MSBuild-node connection and relays the child host's - // messages through the controller protocol. A sandboxed test host cannot and must not open - // the outer build pipe directly. - if (_.Contains(TestHostControllerPidOption, StringComparer.Ordinal)) + // Only sandboxed children must avoid the outer build pipe. Ordinary controller children + // still own the MSBuild lifecycle callbacks and must send module information and results. + if (_.Contains(TestHostControllerPidOption, StringComparer.Ordinal) && IsAppContainer()) { testApplicationBuilder.CommandLine.AddProvider(() => new MSBuildCommandLineProvider()); return; @@ -46,4 +47,29 @@ public static void AddExtensions(ITestApplicationBuilder testApplicationBuilder, testApplicationBuilder.AddMSBuild(); } + + private static bool IsAppContainer() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || Environment.OSVersion.Version < new Version(6, 2)) + { + return false; + } + + const int TokenIsAppContainer = 29; + // GetCurrentProcessToken's borrowed pseudo-handle has TOKEN_QUERY access and needs no cleanup. + IntPtr currentProcessToken = new(-4); + return GetTokenInformation(currentProcessToken, TokenIsAppContainer, out int isAppContainer, sizeof(int), out _) + ? isAppContainer != 0 + : throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + [DefaultDllImportSearchPaths(DllImportSearchPath.System32)] + [DllImport("advapi32.dll", ExactSpelling = true, CallingConvention = CallingConvention.Winapi, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GetTokenInformation( + IntPtr tokenHandle, + int tokenInformationClass, + out int tokenInformation, + int tokenInformationLength, + out int returnLength); } diff --git a/src/Platform/Microsoft.Testing.Platform.MSBuild/Tasks/InvokeTestingPlatformTask.Execution.cs b/src/Platform/Microsoft.Testing.Platform.MSBuild/Tasks/InvokeTestingPlatformTask.Execution.cs index 5f8b38e980..d6cc0eec3b 100644 --- a/src/Platform/Microsoft.Testing.Platform.MSBuild/Tasks/InvokeTestingPlatformTask.Execution.cs +++ b/src/Platform/Microsoft.Testing.Platform.MSBuild/Tasks/InvokeTestingPlatformTask.Execution.cs @@ -195,6 +195,7 @@ private Task HandleRequestAsync(IRequest request) if (_moduleInfo is null) { _moduleInfo = moduleInfo; + _fileSystem.CreateDirectory(_moduleInfo.TestResultFolder); _outputFileName = $"{Path.GetFileNameWithoutExtension(TargetPath.ItemSpec.Trim())}_{TargetFramework.ItemSpec}_{TestArchitecture.ItemSpec}.log"; _outputFileName = Path.Combine(_moduleInfo.TestResultFolder, _outputFileName); Log.LogMessage(MessageImportance.Low, $"Initializing module info and output file '{_outputFileName}'"); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs index 86ae7ecce5..4c850ae8a6 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs @@ -10,6 +10,61 @@ public class MSBuildTests_Test : AcceptanceTestBase { private const string AssetName = "MSBuildTests"; + [TestMethod] + [DataRow("Test", "", true, DisplayName = "TestTarget_NoController_Passing")] + [DataRow("Test", "", false, DisplayName = "TestTarget_NoController_Failing")] + [DataRow("Test", "--report-trx", true, DisplayName = "TestTarget_Trx_Passing")] + [DataRow("Test", "--report-trx", false, DisplayName = "TestTarget_Trx_Failing")] + [DataRow("Test", "--hangdump", true, DisplayName = "TestTarget_HangDump_Passing")] + [DataRow("Test", "--hangdump", false, DisplayName = "TestTarget_HangDump_Failing")] + [DataRow("Test", "--crashdump", true, DisplayName = "TestTarget_CrashDump_Passing")] + [DataRow("Test", "--crashdump", false, DisplayName = "TestTarget_CrashDump_Failing")] + [DataRow("Build,InvokeTestingPlatform", "", true, DisplayName = "InvokeTestingPlatform_NoController_Passing")] + [DataRow("Build,InvokeTestingPlatform", "", false, DisplayName = "InvokeTestingPlatform_NoController_Failing")] + [DataRow("Build,InvokeTestingPlatform", "--report-trx", true, DisplayName = "InvokeTestingPlatform_Trx_Passing")] + [DataRow("Build,InvokeTestingPlatform", "--report-trx", false, DisplayName = "InvokeTestingPlatform_Trx_Failing")] + [DataRow("Build,InvokeTestingPlatform", "--hangdump", true, DisplayName = "InvokeTestingPlatform_HangDump_Passing")] + [DataRow("Build,InvokeTestingPlatform", "--hangdump", false, DisplayName = "InvokeTestingPlatform_HangDump_Failing")] + [DataRow("Build,InvokeTestingPlatform", "--crashdump", true, DisplayName = "InvokeTestingPlatform_CrashDump_Passing")] + [DataRow("Build,InvokeTestingPlatform", "--crashdump", false, DisplayName = "InvokeTestingPlatform_CrashDump_Failing")] + public async Task InvokeTestingPlatform_Target_With_AutoRegistered_Extensions_Should_Report_Test_Results(string target, string extensionOption, bool testSucceeded) + { + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + "MSBuildControllerTests", + ControllerSourceCode + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$AssertValue$", testSucceeded.ToString().ToLowerInvariant())); + string resultsDirectory = Path.Combine(testAsset.TargetAssetPath, "results"); + + DotnetMuxerResult result = await DotnetCli.RunAsync( + $"build -t:{target} -p:TestingPlatformShowTestsFailure=true -p:TestingPlatformCommandLineArguments=\"{extensionOption} --results-directory %22{resultsDirectory}%22\" \"{testAsset.TargetAssetPath}\"", + workingDirectory: testAsset.TargetAssetPath, + failIfReturnValueIsNotZero: false, + useMultithreadedMSBuild: false, + cancellationToken: TestContext.CancellationToken); + + Assert.DoesNotContain("MSBuild extension failed to receive test application information", result.StandardOutput); + Assert.DoesNotContain("MSBuild extension failed to receive test run summary information", result.StandardOutput); + Assert.AreEqual(testSucceeded ? 0 : 1, result.ExitCode, result.ToString()); + result.AssertOutputContains(testSucceeded + ? "Passed! - Failed: 0, Passed: 1, Skipped: 0, Total: 1" + : "Failed! - Failed: 1, Passed: 0, Skipped: 0, Total: 1"); + if (!testSucceeded) + { + result.AssertOutputContains("error test failed: ControllerTest"); + } + + if (extensionOption == "--report-trx") + { + string trxPath = Assert.ContainsSingle(Directory.GetFiles(resultsDirectory, "*.trx")); + XNamespace ns = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + XElement testResult = Assert.ContainsSingle(XDocument.Load(trxPath).Descendants(ns + "UnitTestResult")); + Assert.AreEqual("ControllerTest", (string?)testResult.Attribute("testName")); + Assert.AreEqual(testSucceeded ? "Passed" : "Failed", (string?)testResult.Attribute("outcome")); + } + } + public static string? FormatBuildMatrixEntry(MethodInfo method, object?[]? data) => $"{data![0]},{(Equals(TargetFrameworks.All.ToMSBuildTargetFrameworks(), data[1]) ? "multitfm" : data[1])},{data[2]},{((bool)data[3]! ? "Succeeded" : "Failed")}"; @@ -443,6 +498,36 @@ public async Task InvokeTestingPlatform_Target_Should_Set_DotnetRoot_For_AppHost disabledResult.AssertOutputContains($"[env] {dotnetRootVariableName}="); } + private const string ControllerSourceCode = """ +#file MSBuildControllerTests.csproj + + + $TargetFramework$ + None + true + true + true + + + +#file global.json +{ + "msbuild-sdks": { + "MSTest.Sdk": "$MSTestVersion$" + } +} + +#file Test1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public sealed class ControllerTests +{ + [TestMethod] + public void ControllerTest() => Assert.IsTrue($AssertValue$); +} +"""; + private const string SourceCode = """ #file MSBuild Tests.csproj diff --git a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/InvokeTestingPlatformTaskTests.cs b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/InvokeTestingPlatformTaskTests.cs index 54075f7673..c641f4d52c 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/InvokeTestingPlatformTaskTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/InvokeTestingPlatformTaskTests.cs @@ -1,17 +1,81 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +extern alias PlatformMSBuild; + using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Microsoft.Testing.Platform.Helpers; using Moq; +using ModuleInfoRequest = PlatformMSBuild::Microsoft.Testing.Extensions.MSBuild.Serializers.ModuleInfoRequest; +using RunSummaryInfoRequest = PlatformMSBuild::Microsoft.Testing.Extensions.MSBuild.Serializers.RunSummaryInfoRequest; + namespace Microsoft.Testing.Platform.MSBuild.UnitTests; [TestClass] public sealed class InvokeTestingPlatformTaskTests { + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public void Execute_ModuleInfoCreatesResultsDirectoryBeforeOpeningLog(bool resultsDirectoryExists) + { + using AppHostTaskFixture fixture = new(); + using TestableInvokeTestingPlatformTask task = fixture.CreateTask(new FileSystem()); + string resultsDirectory = Path.Combine(fixture.DotnetRoot, "nested", "results"); + if (resultsDirectoryExists) + { + Directory.CreateDirectory(resultsDirectory); + } + + task.TestArchitecture = new TaskItem("x64"); + task.SimulatedExitCode = 0; + task.OnSimulatedLaunch = current => + { + current.DeliverRequest(new ModuleInfoRequest("framework", "x64", resultsDirectory)); + current.DeliverRequest(new RunSummaryInfoRequest(1, 0, 1, 0, "1s", AllowSkipped: true)); + }; + + Assert.IsTrue(task.Execute()); + Assert.IsTrue(Directory.Exists(resultsDirectory)); + string logPath = Path.Combine(resultsDirectory, "apphost_net9.0_x64.log"); + Assert.AreEqual( + """ + simulated output + + === COMMAND LINE === + simulated command + + + """.ReplaceLineEndings(), + File.ReadAllText(logPath, Encoding.Unicode).ReplaceLineEndings()); + } + + [TestMethod] + public void Execute_FirstModuleOwnsOutputFileAndCompleteHandshakeAllowsSuccess() + { + using AppHostTaskFixture fixture = new(); + using TestableInvokeTestingPlatformTask task = fixture.CreateTask(new FileSystem()); + string originalResultsDirectory = Path.Combine(fixture.DotnetRoot, "original-results"); + string retryResultsDirectory = Path.Combine(fixture.DotnetRoot, "retry-results"); + task.TestArchitecture = new TaskItem("x64"); + task.SimulatedExitCode = 0; + task.OnSimulatedLaunch = current => + { + current.DeliverRequest(new ModuleInfoRequest("first-framework", "first-architecture", originalResultsDirectory)); + current.DeliverRequest(new ModuleInfoRequest("retry-framework", "retry-architecture", retryResultsDirectory)); + current.DeliverRequest(new RunSummaryInfoRequest(1, 0, 1, 0, "1s", AllowSkipped: true)); + }; + + Assert.IsTrue(task.Execute()); + Assert.AreEqual( + Path.Combine(originalResultsDirectory, "apphost_net9.0_x64.log"), + Assert.ContainsSingle(Directory.GetFiles(originalResultsDirectory))); + Assert.IsFalse(Directory.Exists(retryResultsDirectory)); + } + [TestMethod] [DataRow("##[group]Tests: MyAssembly (net9.0)", true)] [DataRow("##[endgroup]", true)] @@ -287,7 +351,13 @@ private sealed class TestableInvokeTestingPlatformTask : InvokeTestingPlatformTa // on the output-logging behavior. [SetsRequiredMembers] public TestableInvokeTestingPlatformTask() - : base(new StubFileSystem()) + : this(new StubFileSystem()) + { + } + + [SetsRequiredMembers] + public TestableInvokeTestingPlatformTask(IFileSystem fileSystem) + : base(fileSystem) { BuildEngine = Mock.Of(); TargetPath = new TaskItem("MyAssembly.dll"); @@ -302,6 +372,28 @@ public TestableInvokeTestingPlatformTask() public void InvokeLogEventsFromTextOutput(string singleLine) => LogEventsFromTextOutput(singleLine, MessageImportance.High); + public int? SimulatedExitCode { get; set; } + + public Action? OnSimulatedLaunch { get; set; } + + protected override int ExecuteTool(string pathToTool, string responseFileCommands, string commandLineCommands) + { + if (SimulatedExitCode is not int exitCode) + { + return base.ExecuteTool(pathToTool, responseFileCommands, commandLineCommands); + } + + LogToolCommand("simulated command"); + LogEventsFromTextOutput("simulated output", MessageImportance.High); + OnSimulatedLaunch?.Invoke(this); + return exitCode; + } + + public void DeliverRequest(object request) + => ((System.Threading.Tasks.Task)typeof(InvokeTestingPlatformTask) + .GetMethod("HandleRequestAsync", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(this, [request])!).GetAwaiter().GetResult(); + public void InvokeAddAppHostDotnetRootEnvironmentVariable() => AddAppHostDotnetRootEnvironmentVariable(); @@ -338,8 +430,8 @@ public AppHostTaskFixture() public string DotnetRoot { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); - public TestableInvokeTestingPlatformTask CreateTask() - => new() + public TestableInvokeTestingPlatformTask CreateTask(IFileSystem? fileSystem = null) + => new(fileSystem ?? new StubFileSystem()) { TargetPath = new TaskItem(Path.Combine(DotnetRoot, $"apphost{_appHostExtension}")), TargetDir = new TaskItem($"{DotnetRoot}{Path.DirectorySeparatorChar}"), diff --git a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/TestingPlatformBuilderHookTests.cs b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/TestingPlatformBuilderHookTests.cs new file mode 100644 index 0000000000..593b7f7f71 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/TestingPlatformBuilderHookTests.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.MSBuild; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.CommandLine; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Extensions.TestHostOrchestrator; +using Microsoft.Testing.Platform.TestHost; +using Microsoft.Testing.Platform.TestHostOrchestrator; + +using Moq; + +namespace Microsoft.Testing.Platform.MSBuild.UnitTests; + +[TestClass] +public sealed class TestingPlatformBuilderHookTests +{ + [TestMethod] + [DataRow(0)] + [DataRow(1)] + [DataRow(2)] + public void AddExtensions_ControllerPidOptionAtAnyPosition_RegistersFullMSBuildIntegration(int position) + { + List> providers = []; + List> hostLifetimes = []; + List> orchestratorLifetimes = []; + List> dataConsumers = []; + List> sessionLifetimes = []; + Mock commandLine = new(MockBehavior.Strict); + Mock testHost = new(MockBehavior.Strict); + Mock orchestrator = new(MockBehavior.Strict); + Mock builder = new(MockBehavior.Strict); + builder.SetupGet(x => x.CommandLine).Returns(commandLine.Object); + builder.SetupGet(x => x.TestHost).Returns(testHost.Object); + builder.SetupGet(x => x.TestHostOrchestrator).Returns(orchestrator.Object); + commandLine.Setup(x => x.AddProvider(It.IsAny>())) + .Callback>(providers.Add); + testHost.Setup(x => x.AddTestHostApplicationLifetime(It.IsAny>())) + .Callback>(hostLifetimes.Add); + orchestrator.Setup(x => x.AddTestHostOrchestratorApplicationLifetime(It.IsAny>())) + .Callback>(orchestratorLifetimes.Add); + testHost.Setup(x => x.AddDataConsumer(It.IsAny>())) + .Callback>(dataConsumers.Add); + testHost.Setup(x => x.AddTestSessionLifetimeHandler(It.IsAny>())) + .Callback>(sessionLifetimes.Add); + string[] arguments = ["other", "123", "last"]; + arguments[position] = "--internal-testhostcontroller-pid"; + + TestingPlatformBuilderHook.AddExtensions(builder.Object, arguments); + + Assert.IsInstanceOfType(Assert.ContainsSingle(providers)()); + Assert.HasCount(1, hostLifetimes); + Assert.HasCount(1, orchestratorLifetimes); + Assert.AreSame(Assert.ContainsSingle(dataConsumers), Assert.ContainsSingle(sessionLifetimes)); + commandLine.VerifyAll(); + testHost.VerifyAll(); + orchestrator.VerifyAll(); + builder.VerifyAll(); + } +}