diff --git a/DeviceRunners.slnx b/DeviceRunners.slnx index bef97b64..9ceb6e5b 100644 --- a/DeviceRunners.slnx +++ b/DeviceRunners.slnx @@ -10,6 +10,7 @@ + @@ -27,6 +28,7 @@ + diff --git a/Directory.Packages.props b/Directory.Packages.props index 4b2180ba..d78ad214 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,6 +12,10 @@ + + + + diff --git a/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj b/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj index 16e2eea0..f8bc1eaf 100644 --- a/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj +++ b/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj @@ -71,17 +71,20 @@ + + + diff --git a/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs b/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs index 8973cff8..2f19194d 100644 --- a/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs +++ b/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs @@ -61,9 +61,11 @@ public static MauiApp CreateMauiApp() .AddTestAssemblies(typeof(DeviceTestingKitApp.MauiLibrary.XunitTests.UnitTests).Assembly) .AddTestAssemblies(typeof(DeviceTestingKitApp.MauiLibrary.Xunit3Tests.UnitTests).Assembly) .AddTestAssemblies(typeof(DeviceTestingKitApp.Library.NUnitTests.UnitTests).Assembly) + .AddTestAssemblies(typeof(DeviceTestingKitApp.Library.MSTestTests.UnitTests).Assembly) .AddXunit() .AddXunit3() - .AddNUnit()); + .AddNUnit() + .AddMSTest()); #if DEBUG builder.Logging.AddDebug(); diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/DeviceTestingKitApp.Library.MSTestTests.csproj b/sample/test/DeviceTestingKitApp.Library.MSTestTests/DeviceTestingKitApp.Library.MSTestTests.csproj new file mode 100644 index 00000000..5997f17d --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/DeviceTestingKitApp.Library.MSTestTests.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + true + + + + + + + + + + + + diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs new file mode 100644 index 00000000..74c479cd --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs @@ -0,0 +1,116 @@ +namespace DeviceTestingKitApp.Library.MSTestTests; + +[TestClass] +public class UnitTests +{ + public static IEnumerable<(int Value, string Name)> GetTestData() + { + yield return (1, "first"); + yield return (2, "second"); + } + + public static IEnumerable<(int Value, string Name)> TestDataProperty => + [ + (1, "first"), + (2, "second") + ]; + + public static IEnumerable<(int Value, string Name)> TestDataField = + [ + (1, "first"), + (2, "second") + ]; + + // Process-wide on purpose: MSTest's [Retry] re-runs the method in the same host, and a fresh + // class instance is created for every attempt, so instance state cannot survive between them. + static int _retryAttempts; + + public TestContext TestContext { get; set; } + + [TestMethod] + public void SuccessfulTest() + { + var value = true; + Assert.IsTrue(value); + } + + [TestMethod] + [Ignore("This test is skipped.")] + public void SkippedTest() + { + } + + [TestMethod] + [TestCategory("ExpectedFailure")] + public void FailingTest() + { + throw new Exception("This is meant to fail."); + } + + [TestMethod] + [TestCategory("Retry")] + [Retry(3)] + public void FlakyTestPassesOnRetry() + { + // Fails on the first attempt and passes on the retry. Microsoft.Testing.Platform reports + // every [Retry] attempt under the same test node UID, tagging the earlier one + // retry.is-superseded; the visual runner and result channels must record only the final + // passing attempt, not the superseded failure. + var attempt = Interlocked.Increment(ref _retryAttempts); + if (attempt == 1) + Assert.Fail("Simulated flaky failure on the first attempt; the retry is expected to pass."); + } + + [TestMethod] + [DataRow(1)] + public void DataRowTest(int value) + { + Assert.AreEqual(1, value); + } + + [TestMethod] + [DataRow(1, 2, 3, 4)] + [DataRow(10, 20)] + public void TestWithParams(params int[] values) + { + Assert.IsTrue(values.Length > 0); + } + + [TestMethod] + [DataRow(42)] + [DataRow("alpha")] + public void Value_RoundTrips(T value) + { +#pragma warning disable MSTEST0032 // Assertion condition is always true + Assert.AreEqual(value, value); +#pragma warning restore MSTEST0032 // Assertion condition is always true + } + + [TestMethod] + [DynamicData(nameof(GetTestData))] + public void TestWithMethod(int value, string name) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [DynamicData(nameof(TestDataProperty))] + public void TestWithProperty(int value, string name) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [DynamicData(nameof(TestDataField))] + public void TestWithField(int value, string name) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + public async Task CooperativeCancellationTest() + { + Assert.IsNotNull(TestContext); + await Task.Delay(1, TestContext.CancellationToken); + } +} diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/Usings.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/Usings.cs new file mode 100644 index 00000000..540383dc --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/Usings.cs @@ -0,0 +1 @@ +global using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/ViewModels/CounterViewModelTests.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/ViewModels/CounterViewModelTests.cs new file mode 100644 index 00000000..e134b1c3 --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/ViewModels/CounterViewModelTests.cs @@ -0,0 +1,75 @@ +using DeviceTestingKitApp.ViewModels; + +namespace DeviceTestingKitApp.Library.MSTestTests.ViewModels; + +[TestClass] +public class CounterViewModelTests +{ + [TestMethod] + public void InitialStateIsCorrect() + { + var vm = new CounterViewModel(); + + Assert.AreEqual(0, vm.Count); + } + + [TestMethod] + public void SingleIncrementIncrementsByOne() + { + var vm = new CounterViewModel(); + + vm.IncrementCommand.Execute(null); + + Assert.AreEqual(1, vm.Count); + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void IncrementsIncrementCorrectly(int executeCount) + { + var vm = new CounterViewModel(); + + for (var i = 0; i < executeCount; i++) + { + vm.IncrementCommand.Execute(null); + } + + Assert.AreEqual(executeCount, vm.Count); + } + + [TestMethod] + public void UpdatingCountPropertyTriggersPropertyChanged() + { + var vm = new CounterViewModel(); + + var eventTrigger = 0; + vm.PropertyChanged += (sender, args) => + { + Assert.AreEqual(nameof(vm.Count), args.PropertyName); + eventTrigger++; + }; + + vm.Count = 5; + + Assert.AreEqual(1, eventTrigger); + } + + [TestMethod] + public void ExecutingCommandTriggersPropertyChanged() + { + var vm = new CounterViewModel(); + + var eventTrigger = 0; + vm.PropertyChanged += (sender, args) => + { + Assert.AreEqual(nameof(vm.Count), args.PropertyName); + eventTrigger++; + }; + + vm.IncrementCommand.Execute(null); + + Assert.AreEqual(1, eventTrigger); + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj b/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj new file mode 100644 index 00000000..2f8170eb --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj @@ -0,0 +1,31 @@ + + + + net9.0;net10.0 + true + + false + false + false + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs new file mode 100644 index 00000000..9c0c7e4b --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs @@ -0,0 +1,108 @@ +using System.Reflection; + +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.ServerMode.Client; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DeviceRunners.VisualRunners.MSTest; + +/// +/// Hosts a Microsoft.Testing.Platform application in server mode in-process and drives it +/// over the platform's JSON-RPC protocol — the same mechanism Visual Studio and VS Code use. +/// +/// The protocol itself (transport, framing, handshake, serialization) comes from the platform's own +/// Microsoft.Testing.Platform.ServerMode.Client.Sources package: +/// owns the loopback listener, launches the application through the callback below, and streams the +/// testing/testUpdates/tests node notifications back as events. Unlike the console host, +/// server mode delivers node updates for a discovery request without executing any tests. +/// +/// +static class MSTestServerModeHost +{ + static readonly Assembly ClientAssembly = typeof(MSTestServerModeHost).Assembly; + static readonly string ClientName = ClientAssembly.GetName().Name!; + static readonly string ClientVersion = + ClientAssembly.GetCustomAttribute()?.InformationalVersion + ?? ClientAssembly.GetName().Version?.ToString() + ?? "0.0.0"; + + /// + /// Discovers every test in without executing any of them. + /// + public static Task DiscoverTestsAsync( + Assembly assembly, + Action onNode, + CancellationToken cancellationToken) => + RunSessionAsync(assembly, (client, token) => client.DiscoverTestsAsync(token), onNode, cancellationToken); + + /// + /// Runs exactly the tests identified by . + /// + public static Task RunTestsAsync( + Assembly assembly, + IReadOnlyCollection testNodeUids, + Action onNode, + CancellationToken cancellationToken) => + RunSessionAsync(assembly, (client, token) => client.RunTestsAsync(testNodeUids, token), onNode, cancellationToken); + + static async Task RunSessionAsync( + Assembly assembly, + Func operation, + Action onNode, + CancellationToken cancellationToken) + { + var options = new MtpServerClientOptions + { + ClientName = ClientName, + ClientVersion = ClientVersion, + }; + + using var client = await MtpServerClient.LaunchInProcessAsync( + (serverArgs, token) => RunServerAsync(assembly, serverArgs), + options, + cancellationToken); + + // The read loop only starts on the first client operation, so subscribing here is enough to + // see every update. Awaiting the operation guarantees all of its handlers have already run. + client.TestNodesUpdated += OnTestNodesUpdated; + + try + { + await client.InitializeAsync(cancellationToken); + + await operation(client, cancellationToken); + + // Ask the server to exit before teardown closes the transport. + await client.ExitAsync(cancellationToken); + } + finally + { + client.TestNodesUpdated -= OnTestNodesUpdated; + + // Tear the hosted application down asynchronously: the synchronous Dispose below then + // returns immediately, which keeps the UI thread clear of the platform watchdogs + // (Android ANR, the iOS watchdog). + await client.ShutdownAsync(); + } + + void OnTestNodesUpdated(object? sender, MtpTestNodeUpdateEventArgs args) + { + foreach (var change in args.Changes) + onNode(change); + } + } + + static async Task RunServerAsync(Assembly assembly, string[] serverArgs) + { + // The client supplies the complete server-mode argument array ('--server jsonrpc' plus the + // loopback client host/port); it must be forwarded verbatim. + var builder = await TestApplication.CreateBuilderAsync(serverArgs); + + builder.AddMSTest(() => new[] { assembly }); + + using var app = await builder.BuildAsync(); + + return await app.RunAsync(); + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyConfiguration.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyConfiguration.cs new file mode 100644 index 00000000..e11567af --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyConfiguration.cs @@ -0,0 +1,5 @@ +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestAssemblyConfiguration : ITestAssemblyConfiguration +{ +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyInfo.cs new file mode 100644 index 00000000..30fc7f7a --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; + +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestAssemblyInfo : ITestAssemblyInfo +{ + readonly ITestAssemblyConfiguration _configuration = new MSTestTestAssemblyConfiguration(); + + public MSTestTestAssemblyInfo(string assemblyFileName, Assembly assembly) + { + AssemblyFileName = assemblyFileName ?? throw new ArgumentNullException(nameof(assemblyFileName)); + Assembly = assembly ?? throw new ArgumentNullException(nameof(assembly)); + } + + public MSTestTestAssemblyInfo(MSTestTestAssemblyInfo testAssembly, IReadOnlyList testCases) + : this(testAssembly.AssemblyFileName, testAssembly.Assembly) + { + TestCases.AddRange(testCases ?? throw new ArgumentNullException(nameof(testCases))); + } + + public string AssemblyFileName { get; } + + /// + /// The runtime that MSTest is hosted against. Kept so the runner + /// can pass it to AddMSTest without re-resolving it from the file name, which is + /// important on platforms where is empty. + /// + public Assembly Assembly { get; } + + ITestAssemblyConfiguration? ITestAssemblyInfo.Configuration => _configuration; + + public List TestCases { get; } = new(); + + IReadOnlyList ITestAssemblyInfo.TestCases => TestCases; +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs new file mode 100644 index 00000000..1d38379f --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs @@ -0,0 +1,91 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestCaseInfo : ITestCaseInfo +{ + public MSTestTestCaseInfo( + MSTestTestAssemblyInfo assembly, + string uid, + string displayName, + string? testClassNamespace, + string? testClassName, + string? testMethodName, + IReadOnlyDictionary> traits) + { + TestAssembly = assembly ?? throw new ArgumentNullException(nameof(assembly)); + Uid = uid ?? throw new ArgumentNullException(nameof(uid)); + DisplayName = displayName ?? throw new ArgumentNullException(nameof(displayName)); + TestClassNamespace = testClassNamespace; + TestClassName = testClassName; + TestMethodName = testMethodName; + Traits = traits ?? new Dictionary>(); + } + + /// + /// Builds a test case from a discovered server-mode node, deriving the class namespace and + /// bare method name from the platform's location.type / location.method fields. + /// + public static MSTestTestCaseInfo FromDiscoveredNode(MSTestTestAssemblyInfo assembly, MtpTestNodeUpdate node) + { + var uid = node.Uid ?? string.Empty; + var className = node.GetLocationType() is { Length: > 0 } locationType ? locationType : null; + var classNamespace = GetNamespace(className); + var methodName = StripParameters(node.GetLocationMethod()); + + return new MSTestTestCaseInfo(assembly, uid, node.DisplayName ?? uid, classNamespace, className, methodName, node.GetTraits()); + } + + static string? GetNamespace(string? className) + { + if (string.IsNullOrEmpty(className)) + return null; + + var index = className!.LastIndexOf('.'); + return index > 0 ? className.Substring(0, index) : null; + } + + static string? StripParameters(string? method) + { + if (string.IsNullOrEmpty(method)) + return method; + + var index = method!.IndexOf('('); + return index > 0 ? method.Substring(0, index) : method; + } + + public MSTestTestAssemblyInfo TestAssembly { get; } + + ITestAssemblyInfo ITestCaseInfo.TestAssembly => TestAssembly; + + public string AssemblyFileName => TestAssembly.AssemblyFileName; + + /// + /// The Microsoft.Testing.Platform test node UID. Used to filter execution to this specific + /// test via the --filter-uid option, and to match result messages back to this case. + /// + public string Uid { get; } + + public string DisplayName { get; } + + public string? TestClassName { get; } + + public string? TestMethodName { get; } + + public string? TestClassNamespace { get; } + + public IReadOnlyDictionary> Traits { get; } + + public MSTestTestResultInfo? Result { get; private set; } + + ITestResultInfo? ITestCaseInfo.Result => Result; + + public event Action? ResultReported; + + public void ReportResult(MSTestTestResultInfo result) + { + Result = result; + + ResultReported?.Invoke(result); + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs new file mode 100644 index 00000000..91171af0 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs @@ -0,0 +1,66 @@ +using System.Reflection; + +using Microsoft.Extensions.Logging; + +using Microsoft.Testing.Platform.ServerMode.Client; + +namespace DeviceRunners.VisualRunners.MSTest; + +public class MSTestTestDiscoverer : ITestDiscoverer +{ + readonly IDiagnosticsManager? _diagnosticsManager; + readonly IReadOnlyList _testAssemblies; + + public MSTestTestDiscoverer(IVisualTestRunnerConfiguration options, IDiagnosticsManager? diagnosticsManager = null, ILogger? logger = null) + { + _diagnosticsManager = diagnosticsManager; + _testAssemblies = options.TestAssemblies.ToArray(); + } + + public async Task> DiscoverAsync(CancellationToken cancellationToken = default) + { + var result = new List(); + + foreach (var assm in _testAssemblies) + { + if (cancellationToken.IsCancellationRequested) + break; + + // Use the logical name (not the file path) so discovery works on platforms + // without filesystem-backed assemblies such as WASM. + var assemblyFileName = assm.GetName().Name + ".dll"; + + try + { + var testAssembly = new MSTestTestAssemblyInfo(assemblyFileName, assm); + var testCases = new List(); + + void OnNode(MtpTestNodeUpdate node) + { + // A discovery request reports each test as an 'action' node in the 'discovered' + // state, without executing anything. + if (node.IsAction() && node.IsDiscovered()) + testCases.Add(MSTestTestCaseInfo.FromDiscoveredNode(testAssembly, node)); + } + + await MSTestServerModeHost.DiscoverTestsAsync(assm, OnNode, cancellationToken); + + if (testCases.Count > 0) + { + testAssembly.TestCases.AddRange(testCases); + result.Add(testAssembly); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _diagnosticsManager?.PostDiagnosticMessage($"Exception discovering tests in assembly '{assemblyFileName}': '{ex.Message}'{Environment.NewLine}{ex}"); + } + } + + return result; + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs new file mode 100644 index 00000000..575f2c13 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs @@ -0,0 +1,48 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestResultInfo : ITestResultInfo +{ + MSTestTestResultInfo(MSTestTestCaseInfo testCase) + { + TestCase = testCase ?? throw new ArgumentNullException(nameof(testCase)); + } + + public MSTestTestCaseInfo TestCase { get; } + + ITestCaseInfo ITestResultInfo.TestCase => TestCase; + + public TestResultStatus Status { get; private init; } + + public TimeSpan Duration { get; private init; } + + public string? Output { get; private init; } + + public string? ErrorMessage { get; private init; } + + public string? ErrorStackTrace { get; private init; } + + public string? SkipReason { get; private init; } + + /// + /// Builds a result from a terminal server-mode node, or returns null when the node + /// does not represent a final test result. + /// + public static MSTestTestResultInfo? TryCreate(MSTestTestCaseInfo testCase, MtpTestNodeUpdate node) + { + if (node.TerminalStatus() is not { } status) + return null; + + return new MSTestTestResultInfo(testCase) + { + Status = status, + Duration = node.Duration(), + Output = node.CombinedOutput(), + // error.message doubles as the skip reason on a skipped node. + SkipReason = status is TestResultStatus.Skipped ? node.ErrorMessage : null, + ErrorMessage = status is TestResultStatus.Failed ? node.ErrorMessage : null, + ErrorStackTrace = status is TestResultStatus.Failed ? node.ErrorStackTrace : null, + }; + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs new file mode 100644 index 00000000..c6f02a26 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs @@ -0,0 +1,106 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + +namespace DeviceRunners.VisualRunners.MSTest; + +public class MSTestTestRunner : ITestRunner +{ + readonly AsyncLock _executionLock = new(); + + readonly IVisualTestRunnerConfiguration _options; + readonly IResultChannelManager? _resultChannelManager; + readonly IDiagnosticsManager? _diagnosticsManager; + + public MSTestTestRunner(IVisualTestRunnerConfiguration options, IResultChannelManager? resultChannelManager = null, IDiagnosticsManager? diagnosticsManager = null) + { + _options = options; + _resultChannelManager = resultChannelManager; + _diagnosticsManager = diagnosticsManager; + } + + public Task RunTestsAsync(IEnumerable testCases, CancellationToken cancellationToken = default) + { + var grouped = testCases + .OfType() + .GroupBy(t => t.TestAssembly) + .Select(g => new MSTestTestAssemblyInfo(g.Key, g.ToList())) + .ToList(); + + return RunTestsAsync(grouped, cancellationToken); + } + + public async Task RunTestsAsync(IEnumerable testAssemblies, CancellationToken cancellationToken = default) + { + using (await _executionLock.LockAsync()) + { + await using var closing = await ResultChannelManagerScope.OpenAsync(_resultChannelManager); + + var mstestAssemblies = testAssemblies.OfType().ToList(); + if (mstestAssemblies.Count == 0) + return; + + // Each assembly runs in its own server-mode session, so there is no benefit to + // overlapping them. + foreach (var assembly in mstestAssemblies) + { + await RunTestsSafe(assembly, cancellationToken); + } + } + } + + async Task RunTestsSafe(MSTestTestAssemblyInfo assembly, CancellationToken cancellationToken) + { + try + { + await RunTests(assembly, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Cancellation is expected — don't log as an error. + } + catch (Exception ex) + { + _diagnosticsManager?.PostDiagnosticMessage( + $"Exception running tests in assembly '{assembly.AssemblyFileName}': '{ex.Message}'{Environment.NewLine}{ex}"); + } + } + + async Task RunTests(MSTestTestAssemblyInfo assemblyInfo, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + return; + + var testCaseLookup = assemblyInfo.TestCases + .GroupBy(tc => tc.Uid) + .ToDictionary(g => g.Key, g => g.First()); + + if (testCaseLookup.Count == 0) + return; + + // Restrict the run to exactly the requested tests by passing their UIDs to the platform. + var testNodeUids = testCaseLookup.Keys.ToList(); + + void OnNode(MtpTestNodeUpdate node) + { + if (!node.IsAction()) + return; + + if (node.Uid is not { } uid || !testCaseLookup.TryGetValue(uid, out var testCase)) + return; + + // MSTest's [Retry] publishes every attempt under the same UID. Ignore the superseded + // earlier attempts so a fail-then-pass retry records only the final pass, rather than a + // failed result followed by a passed one. + if (node.IsSupersededRetry()) + return; + + var result = MSTestTestResultInfo.TryCreate(testCase, node); + if (result is null) + return; + + testCase.ReportResult(result); + _resultChannelManager?.RecordResult(result); + } + + await MSTestServerModeHost.RunTestsAsync(assemblyInfo.Assembly, testNodeUids, OnNode, cancellationToken); + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs new file mode 100644 index 00000000..376a0439 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs @@ -0,0 +1,108 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + +namespace DeviceRunners.VisualRunners.MSTest; + +/// +/// Reads the parts of a server-mode test node that leaves on the raw +/// property bag — the source location and the traits — and classifies the node's execution state. +/// +static class MtpTestNodeUpdateExtensions +{ + /// True for a runnable test ("action"), false for a container ("group"). + public static bool IsAction(this MtpTestNodeUpdate node) => + string.Equals(node.NodeType, "action", StringComparison.Ordinal); + + public static bool IsDiscovered(this MtpTestNodeUpdate node) => + string.Equals(node.ExecutionState, "discovered", StringComparison.Ordinal); + + /// + /// Maps a terminal execution state to its , or returns null + /// when the node is not carrying a final result (still running, discovered, ...). + /// + public static TestResultStatus? TerminalStatus(this MtpTestNodeUpdate node) => node.ExecutionState switch + { + "passed" => TestResultStatus.Passed, + "skipped" => TestResultStatus.Skipped, + // failed / error / timed-out / canceled all surface as a failure. + "failed" or "error" or "timed-out" or "canceled" => TestResultStatus.Failed, + _ => null, + }; + + /// + /// True when a later retry attempt for the same test node supersedes this update, so it is not the + /// test's final outcome. MSTest's [Retry] reports every attempt under the same UID, tagging + /// the earlier ones with the wire property retry.is-superseded. MTP's retry contract says + /// single-result consumers (TRX, JUnit, the process exit code — and this runner) skip these, so a + /// fail-then-pass retry reports only the final pass. + /// + public static bool IsSupersededRetry(this MtpTestNodeUpdate node) => + node.Node.TryGetValue("retry.is-superseded", out var value) && value is true; + + /// + /// The node's reported execution time as a (wire time.duration-ms), + /// or when the node carried no duration. + /// + public static TimeSpan Duration(this MtpTestNodeUpdate node) => + node.DurationInMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : TimeSpan.Zero; + + /// + /// Joins the node's captured standard output and standard error (MSTest's MTP adapter reports + /// TestContext.WriteLine and console writes through these), or returns null when the + /// node carried neither. + /// + public static string? CombinedOutput(this MtpTestNodeUpdate node) + { + var parts = new[] { node.StandardOutput, node.StandardError } + .Where(part => !string.IsNullOrEmpty(part)); + + var combined = string.Join(Environment.NewLine, parts); + return combined.Length == 0 ? null : combined; + } + + /// Fully qualified declaring type name (namespace + type). + public static string? GetLocationType(this MtpTestNodeUpdate node) => + node.GetString("location.type"); + + /// Method name, possibly with a parameter-type list in parentheses. + public static string? GetLocationMethod(this MtpTestNodeUpdate node) => + node.GetString("location.method"); + + /// + /// Reads the node's traits. Each wire element is an object whose property name is the trait key + /// and whose value is the trait value. MSTest serializes [TestProperty("k","v")] as {"k":"v"}, + /// but [TestCategory("x")] as {"x":""} (name as key, empty value). To stay consistent with the + /// NUnit/xunit backends — and so the runner's "Category" filter works — empty-valued traits are + /// exposed as values of the conventional "Category" trait instead of as bare keys. + /// + public static IReadOnlyDictionary> GetTraits(this MtpTestNodeUpdate node) + { + if (!node.Node.TryGetValue("traits", out var traits) || traits is not IEnumerable traitElements) + return new Dictionary>(); + + var result = new Dictionary>(); + foreach (var trait in traitElements.OfType>()) + { + foreach (var property in trait) + { + var rawValue = property.Value as string; + + var (key, value) = string.IsNullOrEmpty(rawValue) + ? ("Category", property.Key) + : (property.Key, rawValue!); + + if (!result.TryGetValue(key, out var values)) + { + values = new List(); + result[key] = values; + } + + values.Add(value); + } + } + + return result.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyList)kvp.Value); + } + + static string? GetString(this MtpTestNodeUpdate node, string propertyName) => + node.Node.TryGetValue(propertyName, out var value) ? value as string : null; +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/VisualTestRunnerConfigurationBuilderExtensions.cs b/src/DeviceRunners.VisualRunners.MSTest/VisualTestRunnerConfigurationBuilderExtensions.cs new file mode 100644 index 00000000..73825ed1 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/VisualTestRunnerConfigurationBuilderExtensions.cs @@ -0,0 +1,13 @@ +using DeviceRunners.VisualRunners.MSTest; + +namespace DeviceRunners.VisualRunners; + +public static class VisualTestRunnerConfigurationBuilderExtensions +{ + public static TBuilder AddMSTest(this TBuilder builder) + where TBuilder : IVisualTestRunnerConfigurationBuilder + { + builder.AddTestPlatform(); + return builder; + } +}