From 32a5a9c2b4af5087ca867f1a8535620fd7c173b8 Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Mon, 13 Jul 2026 22:09:56 +1200 Subject: [PATCH 01/10] Add initial MSTest v4 backend --- DeviceRunners.slnx | 2 + Directory.Packages.props | 3 + .../DeviceTestingKitApp.DeviceTests.csproj | 3 + .../MauiProgram.cs | 4 +- ...ceTestingKitApp.Library.MSTestTests.csproj | 23 ++ .../UnitTests.cs | 25 ++ .../Usings.cs | 1 + .../ViewModels/CounterViewModelTests.cs | 75 ++++++ .../DeviceRunners.VisualRunners.MSTest.csproj | 31 +++ .../MSTestServerModeHost.cs | 250 ++++++++++++++++++ .../MSTestTestAssemblyConfiguration.cs | 5 + .../MSTestTestAssemblyInfo.cs | 35 +++ .../MSTestTestCaseInfo.cs | 88 ++++++ .../MSTestTestDiscoverer.cs | 69 +++++ .../MSTestTestResultInfo.cs | 60 +++++ .../MSTestTestRunner.cs | 105 ++++++++ ...estRunnerConfigurationBuilderExtensions.cs | 13 + .../WireTestNode.cs | 130 +++++++++ 18 files changed, 921 insertions(+), 1 deletion(-) create mode 100644 sample/test/DeviceTestingKitApp.Library.MSTestTests/DeviceTestingKitApp.Library.MSTestTests.csproj create mode 100644 sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs create mode 100644 sample/test/DeviceTestingKitApp.Library.MSTestTests/Usings.cs create mode 100644 sample/test/DeviceTestingKitApp.Library.MSTestTests/ViewModels/CounterViewModelTests.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj create mode 100644 src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyConfiguration.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyInfo.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/VisualTestRunnerConfigurationBuilderExtensions.cs create mode 100644 src/DeviceRunners.VisualRunners.MSTest/WireTestNode.cs diff --git a/DeviceRunners.slnx b/DeviceRunners.slnx index bef97b64..9ceb6e5b 100644 --- a/DeviceRunners.slnx +++ b/DeviceRunners.slnx @@ -10,6 +10,7 @@ + @@ -27,6 +28,7 @@ + diff --git a/Directory.Packages.props b/Directory.Packages.props index 4b2180ba..d7b077c6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,6 +12,9 @@ + + + diff --git a/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj b/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj index 16e2eea0..f8bc1eaf 100644 --- a/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj +++ b/sample/test/DeviceTestingKitApp.DeviceTests/DeviceTestingKitApp.DeviceTests.csproj @@ -71,17 +71,20 @@ + + + diff --git a/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs b/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs index 8973cff8..2f19194d 100644 --- a/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs +++ b/sample/test/DeviceTestingKitApp.DeviceTests/MauiProgram.cs @@ -61,9 +61,11 @@ public static MauiApp CreateMauiApp() .AddTestAssemblies(typeof(DeviceTestingKitApp.MauiLibrary.XunitTests.UnitTests).Assembly) .AddTestAssemblies(typeof(DeviceTestingKitApp.MauiLibrary.Xunit3Tests.UnitTests).Assembly) .AddTestAssemblies(typeof(DeviceTestingKitApp.Library.NUnitTests.UnitTests).Assembly) + .AddTestAssemblies(typeof(DeviceTestingKitApp.Library.MSTestTests.UnitTests).Assembly) .AddXunit() .AddXunit3() - .AddNUnit()); + .AddNUnit() + .AddMSTest()); #if DEBUG builder.Logging.AddDebug(); diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/DeviceTestingKitApp.Library.MSTestTests.csproj b/sample/test/DeviceTestingKitApp.Library.MSTestTests/DeviceTestingKitApp.Library.MSTestTests.csproj new file mode 100644 index 00000000..5997f17d --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/DeviceTestingKitApp.Library.MSTestTests.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + true + + + + + + + + + + + + diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs new file mode 100644 index 00000000..49778ad6 --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs @@ -0,0 +1,25 @@ +namespace DeviceTestingKitApp.Library.MSTestTests; + +[TestClass] +public class UnitTests +{ + [TestMethod] + public void SuccessfulTest() + { + var value = true; + Assert.IsTrue(value); + } + + [TestMethod] + [Ignore("This test is skipped.")] + public void SkippedTest() + { + } + + [TestMethod] + [TestCategory("ExpectedFailure")] + public void FailingTest() + { + throw new Exception("This is meant to fail."); + } +} diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/Usings.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/Usings.cs new file mode 100644 index 00000000..540383dc --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/Usings.cs @@ -0,0 +1 @@ +global using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/ViewModels/CounterViewModelTests.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/ViewModels/CounterViewModelTests.cs new file mode 100644 index 00000000..e134b1c3 --- /dev/null +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/ViewModels/CounterViewModelTests.cs @@ -0,0 +1,75 @@ +using DeviceTestingKitApp.ViewModels; + +namespace DeviceTestingKitApp.Library.MSTestTests.ViewModels; + +[TestClass] +public class CounterViewModelTests +{ + [TestMethod] + public void InitialStateIsCorrect() + { + var vm = new CounterViewModel(); + + Assert.AreEqual(0, vm.Count); + } + + [TestMethod] + public void SingleIncrementIncrementsByOne() + { + var vm = new CounterViewModel(); + + vm.IncrementCommand.Execute(null); + + Assert.AreEqual(1, vm.Count); + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void IncrementsIncrementCorrectly(int executeCount) + { + var vm = new CounterViewModel(); + + for (var i = 0; i < executeCount; i++) + { + vm.IncrementCommand.Execute(null); + } + + Assert.AreEqual(executeCount, vm.Count); + } + + [TestMethod] + public void UpdatingCountPropertyTriggersPropertyChanged() + { + var vm = new CounterViewModel(); + + var eventTrigger = 0; + vm.PropertyChanged += (sender, args) => + { + Assert.AreEqual(nameof(vm.Count), args.PropertyName); + eventTrigger++; + }; + + vm.Count = 5; + + Assert.AreEqual(1, eventTrigger); + } + + [TestMethod] + public void ExecutingCommandTriggersPropertyChanged() + { + var vm = new CounterViewModel(); + + var eventTrigger = 0; + vm.PropertyChanged += (sender, args) => + { + Assert.AreEqual(nameof(vm.Count), args.PropertyName); + eventTrigger++; + }; + + vm.IncrementCommand.Execute(null); + + Assert.AreEqual(1, eventTrigger); + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj b/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj new file mode 100644 index 00000000..80c8f501 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj @@ -0,0 +1,31 @@ + + + + net9.0;net10.0 + true + + false + false + false + + + + + + + + + + + + + + + + + + diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs new file mode 100644 index 00000000..7c35c7f6 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs @@ -0,0 +1,250 @@ +using System.Net; +using System.Net.Sockets; +using System.Reflection; +using System.Text; +using System.Text.Json; + +using Microsoft.Testing.Platform.Builder; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DeviceRunners.VisualRunners.MSTest; + +/// +/// Hosts a Microsoft.Testing.Platform application in server mode in-process and drives it +/// over the platform's JSON-RPC protocol — the same mechanism Visual Studio and VS Code use. +/// +/// The test application is the JSON-RPC server: it connects out over TCP to a client host +/// and port. Here we play the client: we listen on a loopback port, launch the platform in +/// the background, perform the initialize handshake, then issue a discovery or run request +/// and stream back the testing/testUpdates/tests node notifications. Unlike the console +/// host, server mode delivers node updates for a discovery request without executing any tests. +/// +/// +static class MSTestServerModeHost +{ + public const string DiscoverTestsMethod = "testing/discoverTests"; + public const string RunTestsMethod = "testing/runTests"; + + const int InitializeRequestId = 0; + const int OperationRequestId = 1; + + public readonly record struct TestNodeRef(string Uid, string DisplayName); + + public static async Task RunSessionAsync( + Assembly assembly, + string method, + IReadOnlyCollection? tests, + Action onNode, + CancellationToken cancellationToken) + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + + // Launch the platform as a JSON-RPC server that connects back to our listener. + var serverArgs = new[] + { + "--server", "jsonrpc", + "--client-host", "127.0.0.1", + "--client-port", port.ToString(), + }; + + var serverTask = Task.Run(() => RunServerAsync(assembly, serverArgs), cancellationToken); + + try + { + var acceptTask = listener.AcceptTcpClientAsync(cancellationToken).AsTask(); + + // If the server faults before connecting, don't wait for a connection that will + // never arrive — surface the server failure instead. + if (await Task.WhenAny(acceptTask, serverTask) == serverTask) + { + await serverTask; + throw new InvalidOperationException("The MSTest server-mode host exited before establishing a connection."); + } + + using var client = await acceptTask; + using var stream = client.GetStream(); + + await DriveSessionAsync(stream, method, tests, onNode, cancellationToken); + } + finally + { + // Surface any server-side failure (and let it finish shutting down). + await serverTask; + } + } + finally + { + listener.Stop(); + } + } + + static async Task RunServerAsync(Assembly assembly, string[] serverArgs) + { + // Server mode is selected by the '--server' option in the args; a dedicated builder factory + // is no longer needed (and is obsolete). + var builder = await TestApplication.CreateBuilderAsync(serverArgs); + + builder.AddMSTest(() => new[] { assembly }); + + using var app = await builder.BuildAsync(); + + await app.RunAsync(); + } + + static async Task DriveSessionAsync( + Stream stream, + string method, + IReadOnlyCollection? tests, + Action onNode, + CancellationToken cancellationToken) + { + // 1. initialize handshake. + await SendAsync(stream, new + { + jsonrpc = "2.0", + id = InitializeRequestId, + method = "initialize", + @params = new + { + processId = Environment.ProcessId, + clientInfo = new { name = "DeviceRunners.VisualRunners.MSTest", version = "1.0.0" }, + capabilities = new { testing = new { debuggerProvider = false } }, + }, + }, cancellationToken); + + await ReadUntilResponseAsync(stream, InitializeRequestId, onNode: null, cancellationToken); + + // 2. discovery or run request. + object requestParams = tests is null + ? new { runId = Guid.NewGuid().ToString() } + : new + { + runId = Guid.NewGuid().ToString(), + tests = tests.Select(t => new Dictionary + { + ["uid"] = t.Uid, + ["display-name"] = t.DisplayName, + }).ToArray(), + }; + + await SendAsync(stream, new + { + jsonrpc = "2.0", + id = OperationRequestId, + method, + @params = requestParams, + }, cancellationToken); + + // 3. stream node updates until the request completes. + await ReadUntilResponseAsync(stream, OperationRequestId, onNode, cancellationToken); + + // 4. ask the server to exit. + await SendAsync(stream, new { jsonrpc = "2.0", method = "exit" }, cancellationToken); + } + + static async Task ReadUntilResponseAsync(Stream stream, int requestId, Action? onNode, CancellationToken cancellationToken) + { + while (true) + { + var body = await ReadFramedMessageAsync(stream, cancellationToken); + if (body is null) + return; // connection closed + + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + + // A response to our request carries a matching id and a result/error. + if (root.TryGetProperty("id", out var idElement) && + idElement.ValueKind == JsonValueKind.Number && + idElement.GetInt32() == requestId && + (root.TryGetProperty("result", out _) || root.TryGetProperty("error", out _))) + { + return; + } + + if (onNode is not null && + root.TryGetProperty("method", out var methodElement) && + methodElement.ValueKind == JsonValueKind.String && + methodElement.GetString() == "testing/testUpdates/tests") + { + DispatchNodeUpdates(root, onNode); + } + } + } + + static void DispatchNodeUpdates(JsonElement root, Action onNode) + { + if (!root.TryGetProperty("params", out var parameters) || + !parameters.TryGetProperty("changes", out var changes) || + changes.ValueKind != JsonValueKind.Array) + { + return; + } + + foreach (var change in changes.EnumerateArray()) + { + if (change.TryGetProperty("node", out var node) && node.ValueKind == JsonValueKind.Object) + onNode(WireTestNode.FromJson(node)); + } + } + + static async Task SendAsync(Stream stream, object message, CancellationToken cancellationToken) + { + // Default encoder escapes non-ASCII to \uXXXX, so the UTF-8 byte count equals the character + // count the platform's StreamReader.ReadBlockAsync expects from the Content-Length header. + var body = JsonSerializer.SerializeToUtf8Bytes(message); + var header = Encoding.ASCII.GetBytes( + $"Content-Length: {body.Length}\r\nContent-Type: application/testingplatform\r\n\r\n"); + + await stream.WriteAsync(header, cancellationToken); + await stream.WriteAsync(body, cancellationToken); + await stream.FlushAsync(cancellationToken); + } + + static async Task ReadFramedMessageAsync(Stream stream, CancellationToken cancellationToken) + { + var header = new List(64); + var one = new byte[1]; + + while (true) + { + var read = await stream.ReadAsync(one.AsMemory(0, 1), cancellationToken); + if (read == 0) + return header.Count == 0 ? null : throw new EndOfStreamException("Connection closed mid-message."); + + header.Add(one[0]); + + var count = header.Count; + if (count >= 4 && header[count - 4] == '\r' && header[count - 3] == '\n' && header[count - 2] == '\r' && header[count - 1] == '\n') + break; + } + + var contentLength = ParseContentLength(Encoding.ASCII.GetString(header.ToArray())); + if (contentLength < 0) + throw new InvalidDataException("Missing Content-Length header in server-mode message."); + + var body = new byte[contentLength]; + await stream.ReadExactlyAsync(body, cancellationToken); + return body; + } + + static int ParseContentLength(string headers) + { + foreach (var line in headers.Split("\r\n", StringSplitOptions.RemoveEmptyEntries)) + { + const string prefix = "Content-Length:"; + if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && + int.TryParse(line.AsSpan(prefix.Length).Trim(), out var length)) + { + return length; + } + } + + return -1; + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyConfiguration.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyConfiguration.cs new file mode 100644 index 00000000..e11567af --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyConfiguration.cs @@ -0,0 +1,5 @@ +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestAssemblyConfiguration : ITestAssemblyConfiguration +{ +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyInfo.cs new file mode 100644 index 00000000..30fc7f7a --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestAssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; + +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestAssemblyInfo : ITestAssemblyInfo +{ + readonly ITestAssemblyConfiguration _configuration = new MSTestTestAssemblyConfiguration(); + + public MSTestTestAssemblyInfo(string assemblyFileName, Assembly assembly) + { + AssemblyFileName = assemblyFileName ?? throw new ArgumentNullException(nameof(assemblyFileName)); + Assembly = assembly ?? throw new ArgumentNullException(nameof(assembly)); + } + + public MSTestTestAssemblyInfo(MSTestTestAssemblyInfo testAssembly, IReadOnlyList testCases) + : this(testAssembly.AssemblyFileName, testAssembly.Assembly) + { + TestCases.AddRange(testCases ?? throw new ArgumentNullException(nameof(testCases))); + } + + public string AssemblyFileName { get; } + + /// + /// The runtime that MSTest is hosted against. Kept so the runner + /// can pass it to AddMSTest without re-resolving it from the file name, which is + /// important on platforms where is empty. + /// + public Assembly Assembly { get; } + + ITestAssemblyConfiguration? ITestAssemblyInfo.Configuration => _configuration; + + public List TestCases { get; } = new(); + + IReadOnlyList ITestAssemblyInfo.TestCases => TestCases; +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs new file mode 100644 index 00000000..3133f4eb --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs @@ -0,0 +1,88 @@ +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestCaseInfo : ITestCaseInfo +{ + public MSTestTestCaseInfo( + MSTestTestAssemblyInfo assembly, + string uid, + string displayName, + string? testClassNamespace, + string? testClassName, + string? testMethodName, + IReadOnlyDictionary> traits) + { + TestAssembly = assembly ?? throw new ArgumentNullException(nameof(assembly)); + Uid = uid ?? throw new ArgumentNullException(nameof(uid)); + DisplayName = displayName ?? throw new ArgumentNullException(nameof(displayName)); + TestClassNamespace = testClassNamespace; + TestClassName = testClassName; + TestMethodName = testMethodName; + Traits = traits ?? new Dictionary>(); + } + + /// + /// Builds a test case from a discovered server-mode node, deriving the class namespace and + /// bare method name from the platform's location.type / location.method fields. + /// + public static MSTestTestCaseInfo FromDiscoveredNode(MSTestTestAssemblyInfo assembly, WireTestNode node) + { + var className = string.IsNullOrEmpty(node.LocationType) ? null : node.LocationType; + var classNamespace = GetNamespace(className); + var methodName = StripParameters(node.LocationMethod); + + return new MSTestTestCaseInfo(assembly, node.Uid, node.DisplayName, classNamespace, className, methodName, node.Traits); + } + + static string? GetNamespace(string? className) + { + if (string.IsNullOrEmpty(className)) + return null; + + var index = className!.LastIndexOf('.'); + return index > 0 ? className.Substring(0, index) : null; + } + + static string? StripParameters(string? method) + { + if (string.IsNullOrEmpty(method)) + return method; + + var index = method!.IndexOf('('); + return index > 0 ? method.Substring(0, index) : method; + } + + public MSTestTestAssemblyInfo TestAssembly { get; } + + ITestAssemblyInfo ITestCaseInfo.TestAssembly => TestAssembly; + + public string AssemblyFileName => TestAssembly.AssemblyFileName; + + /// + /// The Microsoft.Testing.Platform test node UID. Used to filter execution to this specific + /// test via the --filter-uid option, and to match result messages back to this case. + /// + public string Uid { get; } + + public string DisplayName { get; } + + public string? TestClassName { get; } + + public string? TestMethodName { get; } + + public string? TestClassNamespace { get; } + + public IReadOnlyDictionary> Traits { get; } + + public MSTestTestResultInfo? Result { get; private set; } + + ITestResultInfo? ITestCaseInfo.Result => Result; + + public event Action? ResultReported; + + public void ReportResult(MSTestTestResultInfo result) + { + Result = result; + + ResultReported?.Invoke(result); + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs new file mode 100644 index 00000000..8559ce5e --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs @@ -0,0 +1,69 @@ +using System.Reflection; + +using Microsoft.Extensions.Logging; + +namespace DeviceRunners.VisualRunners.MSTest; + +public class MSTestTestDiscoverer : ITestDiscoverer +{ + readonly IDiagnosticsManager? _diagnosticsManager; + readonly IReadOnlyList _testAssemblies; + + public MSTestTestDiscoverer(IVisualTestRunnerConfiguration options, IDiagnosticsManager? diagnosticsManager = null, ILogger? logger = null) + { + _diagnosticsManager = diagnosticsManager; + _testAssemblies = options.TestAssemblies.ToArray(); + } + + public async Task> DiscoverAsync(CancellationToken cancellationToken = default) + { + var result = new List(); + + foreach (var assm in _testAssemblies) + { + if (cancellationToken.IsCancellationRequested) + break; + + // Use the logical name (not the file path) so discovery works on platforms + // without filesystem-backed assemblies such as WASM. + var assemblyFileName = assm.GetName().Name + ".dll"; + + try + { + var testAssembly = new MSTestTestAssemblyInfo(assemblyFileName, assm); + var testCases = new List(); + + void OnNode(WireTestNode node) + { + // A discovery request reports each test as an 'action' node in the 'discovered' + // state, without executing anything. + if (node.IsAction && node.IsDiscovered) + testCases.Add(MSTestTestCaseInfo.FromDiscoveredNode(testAssembly, node)); + } + + await MSTestServerModeHost.RunSessionAsync( + assm, + MSTestServerModeHost.DiscoverTestsMethod, + tests: null, + OnNode, + cancellationToken); + + if (testCases.Count > 0) + { + testAssembly.TestCases.AddRange(testCases); + result.Add(testAssembly); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _diagnosticsManager?.PostDiagnosticMessage($"Exception discovering tests in assembly '{assemblyFileName}': '{ex.Message}'{Environment.NewLine}{ex}"); + } + } + + return result; + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs new file mode 100644 index 00000000..d0041450 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs @@ -0,0 +1,60 @@ +namespace DeviceRunners.VisualRunners.MSTest; + +class MSTestTestResultInfo : ITestResultInfo +{ + MSTestTestResultInfo(MSTestTestCaseInfo testCase) + { + TestCase = testCase ?? throw new ArgumentNullException(nameof(testCase)); + } + + public MSTestTestCaseInfo TestCase { get; } + + ITestCaseInfo ITestResultInfo.TestCase => TestCase; + + public TestResultStatus Status { get; private init; } + + public TimeSpan Duration { get; private init; } + + public string? Output { get; private init; } + + public string? ErrorMessage { get; private init; } + + public string? ErrorStackTrace { get; private init; } + + public string? SkipReason { get; private init; } + + /// + /// Builds a result from a terminal server-mode node, or returns null when the node + /// does not represent a final test result. + /// + public static MSTestTestResultInfo? TryCreate(MSTestTestCaseInfo testCase, WireTestNode node) + { + if (!node.IsTerminalResult) + return null; + + var duration = node.DurationMs is { } ms ? TimeSpan.FromMilliseconds(ms) : TimeSpan.Zero; + + return node.ExecutionState switch + { + "passed" => new MSTestTestResultInfo(testCase) + { + Status = TestResultStatus.Passed, + Duration = duration, + }, + "skipped" => new MSTestTestResultInfo(testCase) + { + Status = TestResultStatus.Skipped, + Duration = duration, + SkipReason = node.ErrorMessage, + }, + // failed / error / timed-out / canceled all surface as a failure. + _ => new MSTestTestResultInfo(testCase) + { + Status = TestResultStatus.Failed, + Duration = duration, + ErrorMessage = node.ErrorMessage, + ErrorStackTrace = node.ErrorStackTrace, + }, + }; + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs new file mode 100644 index 00000000..17699861 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs @@ -0,0 +1,105 @@ +namespace DeviceRunners.VisualRunners.MSTest; + +public class MSTestTestRunner : ITestRunner +{ + readonly AsyncLock _executionLock = new(); + + readonly IVisualTestRunnerConfiguration _options; + readonly IResultChannelManager? _resultChannelManager; + readonly IDiagnosticsManager? _diagnosticsManager; + + public MSTestTestRunner(IVisualTestRunnerConfiguration options, IResultChannelManager? resultChannelManager = null, IDiagnosticsManager? diagnosticsManager = null) + { + _options = options; + _resultChannelManager = resultChannelManager; + _diagnosticsManager = diagnosticsManager; + } + + public Task RunTestsAsync(IEnumerable testCases, CancellationToken cancellationToken = default) + { + var grouped = testCases + .OfType() + .GroupBy(t => t.TestAssembly) + .Select(g => new MSTestTestAssemblyInfo(g.Key, g.ToList())) + .ToList(); + + return RunTestsAsync(grouped, cancellationToken); + } + + public async Task RunTestsAsync(IEnumerable testAssemblies, CancellationToken cancellationToken = default) + { + using (await _executionLock.LockAsync()) + { + await using var closing = await ResultChannelManagerScope.OpenAsync(_resultChannelManager); + + var mstestAssemblies = testAssemblies.OfType().ToList(); + if (mstestAssemblies.Count == 0) + return; + + // Each assembly runs in its own server-mode session, so there is no benefit to + // overlapping them. + foreach (var assembly in mstestAssemblies) + { + await RunTestsSafe(assembly, cancellationToken); + } + } + } + + async Task RunTestsSafe(MSTestTestAssemblyInfo assembly, CancellationToken cancellationToken) + { + try + { + await RunTests(assembly, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Cancellation is expected — don't log as an error. + } + catch (Exception ex) + { + _diagnosticsManager?.PostDiagnosticMessage( + $"Exception running tests in assembly '{assembly.AssemblyFileName}': '{ex.Message}'{Environment.NewLine}{ex}"); + } + } + + async Task RunTests(MSTestTestAssemblyInfo assemblyInfo, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + return; + + var testCaseLookup = assemblyInfo.TestCases + .GroupBy(tc => tc.Uid) + .ToDictionary(g => g.Key, g => g.First()); + + if (testCaseLookup.Count == 0) + return; + + // Restrict the run to exactly the requested tests by passing their UIDs to the platform. + var tests = testCaseLookup.Values + .Select(tc => new MSTestServerModeHost.TestNodeRef(tc.Uid, tc.DisplayName)) + .ToList(); + + void OnNode(WireTestNode node) + { + if (!node.IsAction) + return; + + if (!testCaseLookup.TryGetValue(node.Uid, out var testCase)) + return; + + var result = MSTestTestResultInfo.TryCreate(testCase, node); + if (result is null) + return; + + testCase.ReportResult(result); + _resultChannelManager?.RecordResult(result); + } + + await MSTestServerModeHost.RunSessionAsync( + assemblyInfo.Assembly, + MSTestServerModeHost.RunTestsMethod, + tests, + OnNode, + cancellationToken); + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/VisualTestRunnerConfigurationBuilderExtensions.cs b/src/DeviceRunners.VisualRunners.MSTest/VisualTestRunnerConfigurationBuilderExtensions.cs new file mode 100644 index 00000000..73825ed1 --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/VisualTestRunnerConfigurationBuilderExtensions.cs @@ -0,0 +1,13 @@ +using DeviceRunners.VisualRunners.MSTest; + +namespace DeviceRunners.VisualRunners; + +public static class VisualTestRunnerConfigurationBuilderExtensions +{ + public static TBuilder AddMSTest(this TBuilder builder) + where TBuilder : IVisualTestRunnerConfigurationBuilder + { + builder.AddTestPlatform(); + return builder; + } +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/WireTestNode.cs b/src/DeviceRunners.VisualRunners.MSTest/WireTestNode.cs new file mode 100644 index 00000000..91dac83a --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/WireTestNode.cs @@ -0,0 +1,130 @@ +using System.Text.Json; + +namespace DeviceRunners.VisualRunners.MSTest; + +/// +/// A test node as it crosses the Microsoft.Testing.Platform server-mode JSON-RPC channel +/// (the node object inside a testing/testUpdates/tests notification). +/// +readonly struct WireTestNode +{ + WireTestNode( + string uid, + string displayName, + string? nodeType, + string? executionState, + string? locationType, + string? locationMethod, + double? durationMs, + string? errorMessage, + string? errorStackTrace, + IReadOnlyDictionary> traits) + { + Uid = uid; + DisplayName = displayName; + NodeType = nodeType; + ExecutionState = executionState; + LocationType = locationType; + LocationMethod = locationMethod; + DurationMs = durationMs; + ErrorMessage = errorMessage; + ErrorStackTrace = errorStackTrace; + Traits = traits; + } + + public string Uid { get; } + public string DisplayName { get; } + + /// "action" for a runnable test, "group" for a container. + public string? NodeType { get; } + + /// discovered / in-progress / passed / skipped / failed / error / timed-out / canceled. + public string? ExecutionState { get; } + + /// Fully qualified declaring type name (namespace + type). + public string? LocationType { get; } + + /// Method name, possibly with a parameter-type list in parentheses. + public string? LocationMethod { get; } + + public double? DurationMs { get; } + public string? ErrorMessage { get; } + public string? ErrorStackTrace { get; } + public IReadOnlyDictionary> Traits { get; } + + public bool IsAction => string.Equals(NodeType, "action", StringComparison.Ordinal); + + public bool IsDiscovered => string.Equals(ExecutionState, "discovered", StringComparison.Ordinal); + + /// True for terminal execution states that carry a final result. + public bool IsTerminalResult => ExecutionState switch + { + "passed" or "skipped" or "failed" or "error" or "timed-out" or "canceled" => true, + _ => false, + }; + + public static WireTestNode FromJson(JsonElement node) + { + var uid = GetString(node, "uid") ?? string.Empty; + var displayName = GetString(node, "display-name") ?? uid; + + return new WireTestNode( + uid, + displayName, + GetString(node, "node-type"), + GetString(node, "execution-state"), + GetString(node, "location.type"), + GetString(node, "location.method"), + GetDouble(node, "time.duration-ms"), + GetString(node, "error.message"), + GetString(node, "error.stacktrace"), + ParseTraits(node)); + } + + static IReadOnlyDictionary> ParseTraits(JsonElement node) + { + if (!node.TryGetProperty("traits", out var traits) || traits.ValueKind != JsonValueKind.Array) + return new Dictionary>(); + + // Each element is an object whose property name is the trait key and whose value is the + // trait value. MSTest serializes [TestProperty("k","v")] as {"k":"v"}, but [TestCategory("x")] + // as {"x":""} (name as key, empty value). To stay consistent with the NUnit/xunit backends — + // and so the runner's "Category" filter works — expose empty-valued traits as values of the + // conventional "Category" trait instead of as bare keys. + var result = new Dictionary>(); + foreach (var trait in traits.EnumerateArray()) + { + if (trait.ValueKind != JsonValueKind.Object) + continue; + + foreach (var property in trait.EnumerateObject()) + { + var rawValue = property.Value.ValueKind == JsonValueKind.String ? property.Value.GetString() : null; + + var (key, value) = string.IsNullOrEmpty(rawValue) + ? ("Category", property.Name) + : (property.Name, rawValue!); + + if (!result.TryGetValue(key, out var values)) + { + values = new List(); + result[key] = values; + } + + values.Add(value); + } + } + + return result.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyList)kvp.Value); + } + + static string? GetString(JsonElement element, string propertyName) => + element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + + static double? GetDouble(JsonElement element, string propertyName) => + element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Number + ? value.GetDouble() + : null; +} From fb60d5f8cfbabce64afe4c9ef99aacb5358c7be1 Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Tue, 21 Jul 2026 11:38:14 +1200 Subject: [PATCH 02/10] Add placeholder test for async co-operative cancellation Mostly check that the TestContext has been set --- .../DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs index 49778ad6..fca4426d 100644 --- a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs @@ -3,6 +3,8 @@ namespace DeviceTestingKitApp.Library.MSTestTests; [TestClass] public class UnitTests { + public TestContext TestContext { get; set; } + [TestMethod] public void SuccessfulTest() { @@ -22,4 +24,11 @@ public void FailingTest() { throw new Exception("This is meant to fail."); } + + [TestMethod] + public async Task CooperativeCancellationTest() + { + Assert.IsNotNull(TestContext); + await Task.Delay(1, TestContext.CancellationToken); + } } From ba3478ec15959a0a650d62429b85fc7546f6ece6 Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Tue, 21 Jul 2026 11:49:22 +1200 Subject: [PATCH 03/10] Add data driven tests --- .../UnitTests.cs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs index fca4426d..b1b6a65a 100644 --- a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs @@ -3,6 +3,24 @@ namespace DeviceTestingKitApp.Library.MSTestTests; [TestClass] public class UnitTests { + public static IEnumerable<(int Value, string Name)> GetTestData() + { + yield return (1, "first"); + yield return (2, "second"); + } + + public static IEnumerable<(int Value, string Name)> TestDataProperty => + [ + (1, "first"), + (2, "second") + ]; + + public static IEnumerable<(int Value, string Name)> TestDataField = + [ + (1, "first"), + (2, "second") + ]; + public TestContext TestContext { get; set; } [TestMethod] @@ -25,6 +43,52 @@ public void FailingTest() throw new Exception("This is meant to fail."); } + [TestMethod] + [DataRow(1)] + public void DataRowTest(int value) + { + Assert.AreEqual(1, value); + } + + [TestMethod] + [DataRow(1, 2, 3, 4)] + [DataRow(10, 20)] + public void TestWithParams(params int[] values) + { + Assert.IsTrue(values.Length > 0); + } + + [TestMethod] + [DataRow(42)] + [DataRow("alpha")] + public void Value_RoundTrips(T value) + { +#pragma warning disable MSTEST0032 // Assertion condition is always true + Assert.AreEqual(value, value); +#pragma warning restore MSTEST0032 // Assertion condition is always true + } + + [TestMethod] + [DynamicData(nameof(GetTestData))] + public void TestWithMethod(int value, string name) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [DynamicData(nameof(TestDataProperty))] + public void TestWithProperty(int value, string name) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [DynamicData(nameof(TestDataField))] + public void TestWithField(int value, string name) + { + Assert.IsTrue(value > 0); + } + [TestMethod] public async Task CooperativeCancellationTest() { From 59e54924dee6436bcaf11b00e2143a214f794af6 Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Tue, 21 Jul 2026 12:01:09 +1200 Subject: [PATCH 04/10] Set client info from assembly data --- .../MSTestServerModeHost.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs index 7c35c7f6..d04bca2c 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs @@ -29,6 +29,13 @@ static class MSTestServerModeHost const int InitializeRequestId = 0; const int OperationRequestId = 1; + static readonly Assembly ClientAssembly = typeof(MSTestServerModeHost).Assembly; + static readonly string ClientName = ClientAssembly.GetName().Name!; + static readonly string ClientVersion = + ClientAssembly.GetCustomAttribute()?.InformationalVersion + ?? ClientAssembly.GetName().Version?.ToString() + ?? "0.0.0"; + public readonly record struct TestNodeRef(string Uid, string DisplayName); public static async Task RunSessionAsync( @@ -112,7 +119,7 @@ static async Task DriveSessionAsync( @params = new { processId = Environment.ProcessId, - clientInfo = new { name = "DeviceRunners.VisualRunners.MSTest", version = "1.0.0" }, + clientInfo = new { name = ClientName, version = ClientVersion }, capabilities = new { testing = new { debuggerProvider = false } }, }, }, cancellationToken); From 11ac561178b579eadf6fb93d0c3059fc4e69a5df Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Tue, 21 Jul 2026 12:13:14 +1200 Subject: [PATCH 05/10] Update MSTest to 4.3.2 --- Directory.Packages.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d7b077c6..2e294f48 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,9 +12,9 @@ - - - + + + From f05b6902f628494aa74b98a1ed895f1835ed816b Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Thu, 3 Sep 2026 13:32:42 +1200 Subject: [PATCH 06/10] Remove redundant InternalsVisibleTo declaration --- .../DeviceRunners.VisualRunners.MSTest.csproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj b/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj index 80c8f501..0ce79281 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj +++ b/src/DeviceRunners.VisualRunners.MSTest/DeviceRunners.VisualRunners.MSTest.csproj @@ -9,10 +9,6 @@ false - - - - + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs index d04bca2c..9c0c7e4b 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestServerModeHost.cs @@ -1,10 +1,7 @@ -using System.Net; -using System.Net.Sockets; using System.Reflection; -using System.Text; -using System.Text.Json; using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.ServerMode.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -14,21 +11,15 @@ namespace DeviceRunners.VisualRunners.MSTest; /// Hosts a Microsoft.Testing.Platform application in server mode in-process and drives it /// over the platform's JSON-RPC protocol — the same mechanism Visual Studio and VS Code use. /// -/// The test application is the JSON-RPC server: it connects out over TCP to a client host -/// and port. Here we play the client: we listen on a loopback port, launch the platform in -/// the background, perform the initialize handshake, then issue a discovery or run request -/// and stream back the testing/testUpdates/tests node notifications. Unlike the console -/// host, server mode delivers node updates for a discovery request without executing any tests. +/// The protocol itself (transport, framing, handshake, serialization) comes from the platform's own +/// Microsoft.Testing.Platform.ServerMode.Client.Sources package: +/// owns the loopback listener, launches the application through the callback below, and streams the +/// testing/testUpdates/tests node notifications back as events. Unlike the console host, +/// server mode delivers node updates for a discovery request without executing any tests. /// /// static class MSTestServerModeHost { - public const string DiscoverTestsMethod = "testing/discoverTests"; - public const string RunTestsMethod = "testing/runTests"; - - const int InitializeRequestId = 0; - const int OperationRequestId = 1; - static readonly Assembly ClientAssembly = typeof(MSTestServerModeHost).Assembly; static readonly string ClientName = ClientAssembly.GetName().Name!; static readonly string ClientVersion = @@ -36,222 +27,82 @@ static class MSTestServerModeHost ?? ClientAssembly.GetName().Version?.ToString() ?? "0.0.0"; - public readonly record struct TestNodeRef(string Uid, string DisplayName); + /// + /// Discovers every test in without executing any of them. + /// + public static Task DiscoverTestsAsync( + Assembly assembly, + Action onNode, + CancellationToken cancellationToken) => + RunSessionAsync(assembly, (client, token) => client.DiscoverTestsAsync(token), onNode, cancellationToken); + + /// + /// Runs exactly the tests identified by . + /// + public static Task RunTestsAsync( + Assembly assembly, + IReadOnlyCollection testNodeUids, + Action onNode, + CancellationToken cancellationToken) => + RunSessionAsync(assembly, (client, token) => client.RunTestsAsync(testNodeUids, token), onNode, cancellationToken); - public static async Task RunSessionAsync( + static async Task RunSessionAsync( Assembly assembly, - string method, - IReadOnlyCollection? tests, - Action onNode, + Func operation, + Action onNode, CancellationToken cancellationToken) { - using var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - try + var options = new MtpServerClientOptions { - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - - // Launch the platform as a JSON-RPC server that connects back to our listener. - var serverArgs = new[] - { - "--server", "jsonrpc", - "--client-host", "127.0.0.1", - "--client-port", port.ToString(), - }; + ClientName = ClientName, + ClientVersion = ClientVersion, + }; - var serverTask = Task.Run(() => RunServerAsync(assembly, serverArgs), cancellationToken); + using var client = await MtpServerClient.LaunchInProcessAsync( + (serverArgs, token) => RunServerAsync(assembly, serverArgs), + options, + cancellationToken); - try - { - var acceptTask = listener.AcceptTcpClientAsync(cancellationToken).AsTask(); + // The read loop only starts on the first client operation, so subscribing here is enough to + // see every update. Awaiting the operation guarantees all of its handlers have already run. + client.TestNodesUpdated += OnTestNodesUpdated; - // If the server faults before connecting, don't wait for a connection that will - // never arrive — surface the server failure instead. - if (await Task.WhenAny(acceptTask, serverTask) == serverTask) - { - await serverTask; - throw new InvalidOperationException("The MSTest server-mode host exited before establishing a connection."); - } + try + { + await client.InitializeAsync(cancellationToken); - using var client = await acceptTask; - using var stream = client.GetStream(); + await operation(client, cancellationToken); - await DriveSessionAsync(stream, method, tests, onNode, cancellationToken); - } - finally - { - // Surface any server-side failure (and let it finish shutting down). - await serverTask; - } + // Ask the server to exit before teardown closes the transport. + await client.ExitAsync(cancellationToken); } finally { - listener.Stop(); - } - } - - static async Task RunServerAsync(Assembly assembly, string[] serverArgs) - { - // Server mode is selected by the '--server' option in the args; a dedicated builder factory - // is no longer needed (and is obsolete). - var builder = await TestApplication.CreateBuilderAsync(serverArgs); - - builder.AddMSTest(() => new[] { assembly }); + client.TestNodesUpdated -= OnTestNodesUpdated; - using var app = await builder.BuildAsync(); - - await app.RunAsync(); - } - - static async Task DriveSessionAsync( - Stream stream, - string method, - IReadOnlyCollection? tests, - Action onNode, - CancellationToken cancellationToken) - { - // 1. initialize handshake. - await SendAsync(stream, new - { - jsonrpc = "2.0", - id = InitializeRequestId, - method = "initialize", - @params = new - { - processId = Environment.ProcessId, - clientInfo = new { name = ClientName, version = ClientVersion }, - capabilities = new { testing = new { debuggerProvider = false } }, - }, - }, cancellationToken); - - await ReadUntilResponseAsync(stream, InitializeRequestId, onNode: null, cancellationToken); - - // 2. discovery or run request. - object requestParams = tests is null - ? new { runId = Guid.NewGuid().ToString() } - : new - { - runId = Guid.NewGuid().ToString(), - tests = tests.Select(t => new Dictionary - { - ["uid"] = t.Uid, - ["display-name"] = t.DisplayName, - }).ToArray(), - }; - - await SendAsync(stream, new - { - jsonrpc = "2.0", - id = OperationRequestId, - method, - @params = requestParams, - }, cancellationToken); - - // 3. stream node updates until the request completes. - await ReadUntilResponseAsync(stream, OperationRequestId, onNode, cancellationToken); - - // 4. ask the server to exit. - await SendAsync(stream, new { jsonrpc = "2.0", method = "exit" }, cancellationToken); - } - - static async Task ReadUntilResponseAsync(Stream stream, int requestId, Action? onNode, CancellationToken cancellationToken) - { - while (true) - { - var body = await ReadFramedMessageAsync(stream, cancellationToken); - if (body is null) - return; // connection closed - - using var document = JsonDocument.Parse(body); - var root = document.RootElement; - - // A response to our request carries a matching id and a result/error. - if (root.TryGetProperty("id", out var idElement) && - idElement.ValueKind == JsonValueKind.Number && - idElement.GetInt32() == requestId && - (root.TryGetProperty("result", out _) || root.TryGetProperty("error", out _))) - { - return; - } - - if (onNode is not null && - root.TryGetProperty("method", out var methodElement) && - methodElement.ValueKind == JsonValueKind.String && - methodElement.GetString() == "testing/testUpdates/tests") - { - DispatchNodeUpdates(root, onNode); - } - } - } - - static void DispatchNodeUpdates(JsonElement root, Action onNode) - { - if (!root.TryGetProperty("params", out var parameters) || - !parameters.TryGetProperty("changes", out var changes) || - changes.ValueKind != JsonValueKind.Array) - { - return; + // Tear the hosted application down asynchronously: the synchronous Dispose below then + // returns immediately, which keeps the UI thread clear of the platform watchdogs + // (Android ANR, the iOS watchdog). + await client.ShutdownAsync(); } - foreach (var change in changes.EnumerateArray()) + void OnTestNodesUpdated(object? sender, MtpTestNodeUpdateEventArgs args) { - if (change.TryGetProperty("node", out var node) && node.ValueKind == JsonValueKind.Object) - onNode(WireTestNode.FromJson(node)); + foreach (var change in args.Changes) + onNode(change); } } - static async Task SendAsync(Stream stream, object message, CancellationToken cancellationToken) - { - // Default encoder escapes non-ASCII to \uXXXX, so the UTF-8 byte count equals the character - // count the platform's StreamReader.ReadBlockAsync expects from the Content-Length header. - var body = JsonSerializer.SerializeToUtf8Bytes(message); - var header = Encoding.ASCII.GetBytes( - $"Content-Length: {body.Length}\r\nContent-Type: application/testingplatform\r\n\r\n"); - - await stream.WriteAsync(header, cancellationToken); - await stream.WriteAsync(body, cancellationToken); - await stream.FlushAsync(cancellationToken); - } - - static async Task ReadFramedMessageAsync(Stream stream, CancellationToken cancellationToken) + static async Task RunServerAsync(Assembly assembly, string[] serverArgs) { - var header = new List(64); - var one = new byte[1]; - - while (true) - { - var read = await stream.ReadAsync(one.AsMemory(0, 1), cancellationToken); - if (read == 0) - return header.Count == 0 ? null : throw new EndOfStreamException("Connection closed mid-message."); - - header.Add(one[0]); - - var count = header.Count; - if (count >= 4 && header[count - 4] == '\r' && header[count - 3] == '\n' && header[count - 2] == '\r' && header[count - 1] == '\n') - break; - } - - var contentLength = ParseContentLength(Encoding.ASCII.GetString(header.ToArray())); - if (contentLength < 0) - throw new InvalidDataException("Missing Content-Length header in server-mode message."); + // The client supplies the complete server-mode argument array ('--server jsonrpc' plus the + // loopback client host/port); it must be forwarded verbatim. + var builder = await TestApplication.CreateBuilderAsync(serverArgs); - var body = new byte[contentLength]; - await stream.ReadExactlyAsync(body, cancellationToken); - return body; - } + builder.AddMSTest(() => new[] { assembly }); - static int ParseContentLength(string headers) - { - foreach (var line in headers.Split("\r\n", StringSplitOptions.RemoveEmptyEntries)) - { - const string prefix = "Content-Length:"; - if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && - int.TryParse(line.AsSpan(prefix.Length).Trim(), out var length)) - { - return length; - } - } + using var app = await builder.BuildAsync(); - return -1; + return await app.RunAsync(); } } diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs index 3133f4eb..1d38379f 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestCaseInfo.cs @@ -1,3 +1,5 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + namespace DeviceRunners.VisualRunners.MSTest; class MSTestTestCaseInfo : ITestCaseInfo @@ -24,13 +26,14 @@ public MSTestTestCaseInfo( /// Builds a test case from a discovered server-mode node, deriving the class namespace and /// bare method name from the platform's location.type / location.method fields. /// - public static MSTestTestCaseInfo FromDiscoveredNode(MSTestTestAssemblyInfo assembly, WireTestNode node) + public static MSTestTestCaseInfo FromDiscoveredNode(MSTestTestAssemblyInfo assembly, MtpTestNodeUpdate node) { - var className = string.IsNullOrEmpty(node.LocationType) ? null : node.LocationType; + var uid = node.Uid ?? string.Empty; + var className = node.GetLocationType() is { Length: > 0 } locationType ? locationType : null; var classNamespace = GetNamespace(className); - var methodName = StripParameters(node.LocationMethod); + var methodName = StripParameters(node.GetLocationMethod()); - return new MSTestTestCaseInfo(assembly, node.Uid, node.DisplayName, classNamespace, className, methodName, node.Traits); + return new MSTestTestCaseInfo(assembly, uid, node.DisplayName ?? uid, classNamespace, className, methodName, node.GetTraits()); } static string? GetNamespace(string? className) diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs index 8559ce5e..91171af0 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestDiscoverer.cs @@ -2,6 +2,8 @@ using Microsoft.Extensions.Logging; +using Microsoft.Testing.Platform.ServerMode.Client; + namespace DeviceRunners.VisualRunners.MSTest; public class MSTestTestDiscoverer : ITestDiscoverer @@ -33,20 +35,15 @@ public async Task> DiscoverAsync(CancellationTo var testAssembly = new MSTestTestAssemblyInfo(assemblyFileName, assm); var testCases = new List(); - void OnNode(WireTestNode node) + void OnNode(MtpTestNodeUpdate node) { // A discovery request reports each test as an 'action' node in the 'discovered' // state, without executing anything. - if (node.IsAction && node.IsDiscovered) + if (node.IsAction() && node.IsDiscovered()) testCases.Add(MSTestTestCaseInfo.FromDiscoveredNode(testAssembly, node)); } - await MSTestServerModeHost.RunSessionAsync( - assm, - MSTestServerModeHost.DiscoverTestsMethod, - tests: null, - OnNode, - cancellationToken); + await MSTestServerModeHost.DiscoverTestsAsync(assm, OnNode, cancellationToken); if (testCases.Count > 0) { diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs index d0041450..18363b6d 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs @@ -1,3 +1,5 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + namespace DeviceRunners.VisualRunners.MSTest; class MSTestTestResultInfo : ITestResultInfo @@ -27,12 +29,12 @@ class MSTestTestResultInfo : ITestResultInfo /// Builds a result from a terminal server-mode node, or returns null when the node /// does not represent a final test result. /// - public static MSTestTestResultInfo? TryCreate(MSTestTestCaseInfo testCase, WireTestNode node) + public static MSTestTestResultInfo? TryCreate(MSTestTestCaseInfo testCase, MtpTestNodeUpdate node) { - if (!node.IsTerminalResult) + if (!node.IsTerminalResult()) return null; - var duration = node.DurationMs is { } ms ? TimeSpan.FromMilliseconds(ms) : TimeSpan.Zero; + var duration = node.DurationInMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : TimeSpan.Zero; return node.ExecutionState switch { diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs index 17699861..13a21854 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs @@ -1,3 +1,5 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + namespace DeviceRunners.VisualRunners.MSTest; public class MSTestTestRunner : ITestRunner @@ -75,16 +77,14 @@ async Task RunTests(MSTestTestAssemblyInfo assemblyInfo, CancellationToken cance return; // Restrict the run to exactly the requested tests by passing their UIDs to the platform. - var tests = testCaseLookup.Values - .Select(tc => new MSTestServerModeHost.TestNodeRef(tc.Uid, tc.DisplayName)) - .ToList(); + var testNodeUids = testCaseLookup.Keys.ToList(); - void OnNode(WireTestNode node) + void OnNode(MtpTestNodeUpdate node) { - if (!node.IsAction) + if (!node.IsAction()) return; - if (!testCaseLookup.TryGetValue(node.Uid, out var testCase)) + if (node.Uid is not { } uid || !testCaseLookup.TryGetValue(uid, out var testCase)) return; var result = MSTestTestResultInfo.TryCreate(testCase, node); @@ -95,11 +95,6 @@ void OnNode(WireTestNode node) _resultChannelManager?.RecordResult(result); } - await MSTestServerModeHost.RunSessionAsync( - assemblyInfo.Assembly, - MSTestServerModeHost.RunTestsMethod, - tests, - OnNode, - cancellationToken); + await MSTestServerModeHost.RunTestsAsync(assemblyInfo.Assembly, testNodeUids, OnNode, cancellationToken); } } diff --git a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs new file mode 100644 index 00000000..a3b8f23a --- /dev/null +++ b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs @@ -0,0 +1,71 @@ +using Microsoft.Testing.Platform.ServerMode.Client; + +namespace DeviceRunners.VisualRunners.MSTest; + +/// +/// Reads the parts of a server-mode test node that leaves on the raw +/// property bag — the source location and the traits — and classifies the node's execution state. +/// +static class MtpTestNodeUpdateExtensions +{ + /// True for a runnable test ("action"), false for a container ("group"). + public static bool IsAction(this MtpTestNodeUpdate node) => + string.Equals(node.NodeType, "action", StringComparison.Ordinal); + + public static bool IsDiscovered(this MtpTestNodeUpdate node) => + string.Equals(node.ExecutionState, "discovered", StringComparison.Ordinal); + + /// True for terminal execution states that carry a final result. + public static bool IsTerminalResult(this MtpTestNodeUpdate node) => node.ExecutionState switch + { + "passed" or "skipped" or "failed" or "error" or "timed-out" or "canceled" => true, + _ => false, + }; + + /// Fully qualified declaring type name (namespace + type). + public static string? GetLocationType(this MtpTestNodeUpdate node) => + node.GetString("location.type"); + + /// Method name, possibly with a parameter-type list in parentheses. + public static string? GetLocationMethod(this MtpTestNodeUpdate node) => + node.GetString("location.method"); + + /// + /// Reads the node's traits. Each wire element is an object whose property name is the trait key + /// and whose value is the trait value. MSTest serializes [TestProperty("k","v")] as {"k":"v"}, + /// but [TestCategory("x")] as {"x":""} (name as key, empty value). To stay consistent with the + /// NUnit/xunit backends — and so the runner's "Category" filter works — empty-valued traits are + /// exposed as values of the conventional "Category" trait instead of as bare keys. + /// + public static IReadOnlyDictionary> GetTraits(this MtpTestNodeUpdate node) + { + if (!node.Node.TryGetValue("traits", out var traits) || traits is not IEnumerable traitElements) + return new Dictionary>(); + + var result = new Dictionary>(); + foreach (var trait in traitElements.OfType>()) + { + foreach (var property in trait) + { + var rawValue = property.Value as string; + + var (key, value) = string.IsNullOrEmpty(rawValue) + ? ("Category", property.Key) + : (property.Key, rawValue!); + + if (!result.TryGetValue(key, out var values)) + { + values = new List(); + result[key] = values; + } + + values.Add(value); + } + } + + return result.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyList)kvp.Value); + } + + static string? GetString(this MtpTestNodeUpdate node, string propertyName) => + node.Node.TryGetValue(propertyName, out var value) ? value as string : null; +} diff --git a/src/DeviceRunners.VisualRunners.MSTest/WireTestNode.cs b/src/DeviceRunners.VisualRunners.MSTest/WireTestNode.cs deleted file mode 100644 index 91dac83a..00000000 --- a/src/DeviceRunners.VisualRunners.MSTest/WireTestNode.cs +++ /dev/null @@ -1,130 +0,0 @@ -using System.Text.Json; - -namespace DeviceRunners.VisualRunners.MSTest; - -/// -/// A test node as it crosses the Microsoft.Testing.Platform server-mode JSON-RPC channel -/// (the node object inside a testing/testUpdates/tests notification). -/// -readonly struct WireTestNode -{ - WireTestNode( - string uid, - string displayName, - string? nodeType, - string? executionState, - string? locationType, - string? locationMethod, - double? durationMs, - string? errorMessage, - string? errorStackTrace, - IReadOnlyDictionary> traits) - { - Uid = uid; - DisplayName = displayName; - NodeType = nodeType; - ExecutionState = executionState; - LocationType = locationType; - LocationMethod = locationMethod; - DurationMs = durationMs; - ErrorMessage = errorMessage; - ErrorStackTrace = errorStackTrace; - Traits = traits; - } - - public string Uid { get; } - public string DisplayName { get; } - - /// "action" for a runnable test, "group" for a container. - public string? NodeType { get; } - - /// discovered / in-progress / passed / skipped / failed / error / timed-out / canceled. - public string? ExecutionState { get; } - - /// Fully qualified declaring type name (namespace + type). - public string? LocationType { get; } - - /// Method name, possibly with a parameter-type list in parentheses. - public string? LocationMethod { get; } - - public double? DurationMs { get; } - public string? ErrorMessage { get; } - public string? ErrorStackTrace { get; } - public IReadOnlyDictionary> Traits { get; } - - public bool IsAction => string.Equals(NodeType, "action", StringComparison.Ordinal); - - public bool IsDiscovered => string.Equals(ExecutionState, "discovered", StringComparison.Ordinal); - - /// True for terminal execution states that carry a final result. - public bool IsTerminalResult => ExecutionState switch - { - "passed" or "skipped" or "failed" or "error" or "timed-out" or "canceled" => true, - _ => false, - }; - - public static WireTestNode FromJson(JsonElement node) - { - var uid = GetString(node, "uid") ?? string.Empty; - var displayName = GetString(node, "display-name") ?? uid; - - return new WireTestNode( - uid, - displayName, - GetString(node, "node-type"), - GetString(node, "execution-state"), - GetString(node, "location.type"), - GetString(node, "location.method"), - GetDouble(node, "time.duration-ms"), - GetString(node, "error.message"), - GetString(node, "error.stacktrace"), - ParseTraits(node)); - } - - static IReadOnlyDictionary> ParseTraits(JsonElement node) - { - if (!node.TryGetProperty("traits", out var traits) || traits.ValueKind != JsonValueKind.Array) - return new Dictionary>(); - - // Each element is an object whose property name is the trait key and whose value is the - // trait value. MSTest serializes [TestProperty("k","v")] as {"k":"v"}, but [TestCategory("x")] - // as {"x":""} (name as key, empty value). To stay consistent with the NUnit/xunit backends — - // and so the runner's "Category" filter works — expose empty-valued traits as values of the - // conventional "Category" trait instead of as bare keys. - var result = new Dictionary>(); - foreach (var trait in traits.EnumerateArray()) - { - if (trait.ValueKind != JsonValueKind.Object) - continue; - - foreach (var property in trait.EnumerateObject()) - { - var rawValue = property.Value.ValueKind == JsonValueKind.String ? property.Value.GetString() : null; - - var (key, value) = string.IsNullOrEmpty(rawValue) - ? ("Category", property.Name) - : (property.Name, rawValue!); - - if (!result.TryGetValue(key, out var values)) - { - values = new List(); - result[key] = values; - } - - values.Add(value); - } - } - - return result.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyList)kvp.Value); - } - - static string? GetString(JsonElement element, string propertyName) => - element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String - ? value.GetString() - : null; - - static double? GetDouble(JsonElement element, string propertyName) => - element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Number - ? value.GetDouble() - : null; -} From e8cfc89b455c607049d1b637ee4fd9ff1d26eb89 Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Mon, 7 Sep 2026 08:53:04 +1200 Subject: [PATCH 08/10] Ignore Superseded retries --- .../UnitTests.cs | 18 ++++++++++++++++++ .../MSTestTestRunner.cs | 6 ++++++ .../MtpTestNodeUpdateExtensions.cs | 10 ++++++++++ 3 files changed, 34 insertions(+) diff --git a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs index b1b6a65a..74c479cd 100644 --- a/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs +++ b/sample/test/DeviceTestingKitApp.Library.MSTestTests/UnitTests.cs @@ -21,6 +21,10 @@ public class UnitTests (2, "second") ]; + // Process-wide on purpose: MSTest's [Retry] re-runs the method in the same host, and a fresh + // class instance is created for every attempt, so instance state cannot survive between them. + static int _retryAttempts; + public TestContext TestContext { get; set; } [TestMethod] @@ -43,6 +47,20 @@ public void FailingTest() throw new Exception("This is meant to fail."); } + [TestMethod] + [TestCategory("Retry")] + [Retry(3)] + public void FlakyTestPassesOnRetry() + { + // Fails on the first attempt and passes on the retry. Microsoft.Testing.Platform reports + // every [Retry] attempt under the same test node UID, tagging the earlier one + // retry.is-superseded; the visual runner and result channels must record only the final + // passing attempt, not the superseded failure. + var attempt = Interlocked.Increment(ref _retryAttempts); + if (attempt == 1) + Assert.Fail("Simulated flaky failure on the first attempt; the retry is expected to pass."); + } + [TestMethod] [DataRow(1)] public void DataRowTest(int value) diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs index 13a21854..c6f02a26 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestRunner.cs @@ -87,6 +87,12 @@ void OnNode(MtpTestNodeUpdate node) if (node.Uid is not { } uid || !testCaseLookup.TryGetValue(uid, out var testCase)) return; + // MSTest's [Retry] publishes every attempt under the same UID. Ignore the superseded + // earlier attempts so a fail-then-pass retry records only the final pass, rather than a + // failed result followed by a passed one. + if (node.IsSupersededRetry()) + return; + var result = MSTestTestResultInfo.TryCreate(testCase, node); if (result is null) return; diff --git a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs index a3b8f23a..5f0c63bb 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs @@ -22,6 +22,16 @@ public static bool IsDiscovered(this MtpTestNodeUpdate node) => _ => false, }; + /// + /// True when a later retry attempt for the same test node supersedes this update, so it is not the + /// test's final outcome. MSTest's [Retry] reports every attempt under the same UID, tagging + /// the earlier ones with the wire property retry.is-superseded. MTP's retry contract says + /// single-result consumers (TRX, JUnit, the process exit code — and this runner) skip these, so a + /// fail-then-pass retry reports only the final pass. + /// + public static bool IsSupersededRetry(this MtpTestNodeUpdate node) => + node.Node.TryGetValue("retry.is-superseded", out var value) && value is true; + /// Fully qualified declaring type name (namespace + type). public static string? GetLocationType(this MtpTestNodeUpdate node) => node.GetString("location.type"); From f0be312b4997a0455f18717b4d21e6a62cfac994 Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Mon, 7 Sep 2026 08:57:02 +1200 Subject: [PATCH 09/10] Populate MSTestTestResultInfo.Output --- .../MSTestTestResultInfo.cs | 4 ++++ .../MtpTestNodeUpdateExtensions.cs | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs index 18363b6d..21285278 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs @@ -35,6 +35,7 @@ class MSTestTestResultInfo : ITestResultInfo return null; var duration = node.DurationInMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : TimeSpan.Zero; + var output = node.CombinedOutput(); return node.ExecutionState switch { @@ -42,11 +43,13 @@ class MSTestTestResultInfo : ITestResultInfo { Status = TestResultStatus.Passed, Duration = duration, + Output = output, }, "skipped" => new MSTestTestResultInfo(testCase) { Status = TestResultStatus.Skipped, Duration = duration, + Output = output, SkipReason = node.ErrorMessage, }, // failed / error / timed-out / canceled all surface as a failure. @@ -54,6 +57,7 @@ class MSTestTestResultInfo : ITestResultInfo { Status = TestResultStatus.Failed, Duration = duration, + Output = output, ErrorMessage = node.ErrorMessage, ErrorStackTrace = node.ErrorStackTrace, }, diff --git a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs index 5f0c63bb..a14f75b3 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs @@ -32,6 +32,20 @@ public static bool IsDiscovered(this MtpTestNodeUpdate node) => public static bool IsSupersededRetry(this MtpTestNodeUpdate node) => node.Node.TryGetValue("retry.is-superseded", out var value) && value is true; + /// + /// Joins the node's captured standard output and standard error (MSTest's MTP adapter reports + /// TestContext.WriteLine and console writes through these), or returns null when the + /// node carried neither. + /// + public static string? CombinedOutput(this MtpTestNodeUpdate node) + { + var parts = new[] { node.StandardOutput, node.StandardError } + .Where(part => !string.IsNullOrEmpty(part)); + + var combined = string.Join(Environment.NewLine, parts); + return combined.Length == 0 ? null : combined; + } + /// Fully qualified declaring type name (namespace + type). public static string? GetLocationType(this MtpTestNodeUpdate node) => node.GetString("location.type"); From 0195446ff5b068a7ae66748bad314988e1c3379e Mon Sep 17 00:00:00 2001 From: Jeremy Powell Date: Mon, 7 Sep 2026 12:30:52 +1200 Subject: [PATCH 10/10] Simplify result construction --- .../MSTestTestResultInfo.cs | 36 +++++-------------- .../MtpTestNodeUpdateExtensions.cs | 21 ++++++++--- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs index 21285278..575f2c13 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MSTestTestResultInfo.cs @@ -31,36 +31,18 @@ class MSTestTestResultInfo : ITestResultInfo /// public static MSTestTestResultInfo? TryCreate(MSTestTestCaseInfo testCase, MtpTestNodeUpdate node) { - if (!node.IsTerminalResult()) + if (node.TerminalStatus() is not { } status) return null; - var duration = node.DurationInMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : TimeSpan.Zero; - var output = node.CombinedOutput(); - - return node.ExecutionState switch + return new MSTestTestResultInfo(testCase) { - "passed" => new MSTestTestResultInfo(testCase) - { - Status = TestResultStatus.Passed, - Duration = duration, - Output = output, - }, - "skipped" => new MSTestTestResultInfo(testCase) - { - Status = TestResultStatus.Skipped, - Duration = duration, - Output = output, - SkipReason = node.ErrorMessage, - }, - // failed / error / timed-out / canceled all surface as a failure. - _ => new MSTestTestResultInfo(testCase) - { - Status = TestResultStatus.Failed, - Duration = duration, - Output = output, - ErrorMessage = node.ErrorMessage, - ErrorStackTrace = node.ErrorStackTrace, - }, + Status = status, + Duration = node.Duration(), + Output = node.CombinedOutput(), + // error.message doubles as the skip reason on a skipped node. + SkipReason = status is TestResultStatus.Skipped ? node.ErrorMessage : null, + ErrorMessage = status is TestResultStatus.Failed ? node.ErrorMessage : null, + ErrorStackTrace = status is TestResultStatus.Failed ? node.ErrorStackTrace : null, }; } } diff --git a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs index a14f75b3..376a0439 100644 --- a/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs +++ b/src/DeviceRunners.VisualRunners.MSTest/MtpTestNodeUpdateExtensions.cs @@ -15,11 +15,17 @@ public static bool IsAction(this MtpTestNodeUpdate node) => public static bool IsDiscovered(this MtpTestNodeUpdate node) => string.Equals(node.ExecutionState, "discovered", StringComparison.Ordinal); - /// True for terminal execution states that carry a final result. - public static bool IsTerminalResult(this MtpTestNodeUpdate node) => node.ExecutionState switch + /// + /// Maps a terminal execution state to its , or returns null + /// when the node is not carrying a final result (still running, discovered, ...). + /// + public static TestResultStatus? TerminalStatus(this MtpTestNodeUpdate node) => node.ExecutionState switch { - "passed" or "skipped" or "failed" or "error" or "timed-out" or "canceled" => true, - _ => false, + "passed" => TestResultStatus.Passed, + "skipped" => TestResultStatus.Skipped, + // failed / error / timed-out / canceled all surface as a failure. + "failed" or "error" or "timed-out" or "canceled" => TestResultStatus.Failed, + _ => null, }; /// @@ -32,6 +38,13 @@ public static bool IsDiscovered(this MtpTestNodeUpdate node) => public static bool IsSupersededRetry(this MtpTestNodeUpdate node) => node.Node.TryGetValue("retry.is-superseded", out var value) && value is true; + /// + /// The node's reported execution time as a (wire time.duration-ms), + /// or when the node carried no duration. + /// + public static TimeSpan Duration(this MtpTestNodeUpdate node) => + node.DurationInMilliseconds is { } ms ? TimeSpan.FromMilliseconds(ms) : TimeSpan.Zero; + /// /// Joins the node's captured standard output and standard error (MSTest's MTP adapter reports /// TestContext.WriteLine and console writes through these), or returns null when the