diff --git a/.github/scripts/compare_perf_timings.py b/.github/scripts/compare_perf_timings.py new file mode 100644 index 0000000000..2656e86c8b --- /dev/null +++ b/.github/scripts/compare_perf_timings.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 + +"""Compare performance timings with a rolling baseline.""" + +import argparse +import json +import math +import re +import statistics +from datetime import datetime, timezone +from pathlib import Path + + +SCHEMA_VERSION = 1 +TIME_PATTERN = re.compile( + r"^(?:(?P\d+)\.)?(?P\d{2}):(?P\d{2}):" + r"(?P\d{2})(?:\.(?P\d{1,7}))?$" +) + + +def parse_duration(value: str) -> float: + match = TIME_PATTERN.fullmatch(value) + if match is None: + raise ValueError(f"Invalid TimeSpan value: {value!r}") + + days = int(match.group("days") or 0) + hours = int(match.group("hours")) + minutes = int(match.group("minutes")) + seconds = int(match.group("seconds")) + fraction = (match.group("fraction") or "").ljust(7, "0") + ticks = int(fraction or 0) + return days * 86400 + hours * 3600 + minutes * 60 + seconds + ticks / 10_000_000 + + +def load_current_results(current_dir: Path) -> tuple[dict[str, dict[str, float]], int]: + pipelines: dict[str, dict[str, float]] = {} + processor_counts: set[int] = set() + result_files = sorted(current_dir.rglob("Result.json")) + if not result_files: + raise ValueError(f"No Result.json files found under {current_dir}") + + for result_file in result_files: + report = json.loads(result_file.read_text(encoding="utf-8-sig")) + if not isinstance(report, dict): + raise ValueError(f"{result_file} does not contain a named performance report") + + pipeline_name = report.get("PipelineName") + measurements = report.get("Measurements") + if not isinstance(pipeline_name, str) or not pipeline_name: + raise ValueError(f"{result_file} has no PipelineName") + if pipeline_name in pipelines: + raise ValueError(f"Duplicate performance report for {pipeline_name}") + if not isinstance(measurements, list) or not measurements: + raise ValueError(f"{result_file} has no Measurements") + + elapsed_times: list[float] = [] + processor_times: list[float] = [] + for measurement in measurements: + if not isinstance(measurement, dict): + raise ValueError(f"{result_file} contains an invalid measurement") + + elapsed_times.append(parse_duration(measurement["ElapsedTime"])) + processor_times.append(parse_duration(measurement["TotalProcessorTime"])) + processor_counts.add(int(measurement["ProcessorCount"])) + + pipelines[pipeline_name] = { + "elapsedTimeSeconds": statistics.median(elapsed_times), + "totalProcessorTimeSeconds": statistics.median(processor_times), + } + + if len(processor_counts) != 1: + raise ValueError( + f"Expected one processor count across current results, found {sorted(processor_counts)}" + ) + + return pipelines, processor_counts.pop() + + +def load_baseline(path: Path) -> list[dict]: + if not path.is_file(): + return [] + + try: + baseline = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(baseline, dict): + raise ValueError("Baseline root must be an object") + if baseline.get("schemaVersion") != SCHEMA_VERSION: + raise ValueError( + f"Unsupported baseline schema version: {baseline.get('schemaVersion')!r}" + ) + + runs = baseline.get("runs") + if not isinstance(runs, list): + raise ValueError("Baseline runs must be an array") + for run_index, run in enumerate(runs): + if not isinstance(run, dict): + raise ValueError(f"Baseline run {run_index} must be an object") + + processor_count = run.get("processorCount") + if not isinstance(processor_count, int) or processor_count <= 0: + raise ValueError( + f"Baseline run {run_index} has an invalid processor count" + ) + + pipelines = run.get("pipelines") + if not isinstance(pipelines, dict): + raise ValueError(f"Baseline run {run_index} pipelines must be an object") + for pipeline_name, metrics in pipelines.items(): + if not isinstance(pipeline_name, str) or not pipeline_name: + raise ValueError( + f"Baseline run {run_index} has an invalid pipeline name" + ) + if not isinstance(metrics, dict): + raise ValueError( + f"Baseline metrics for {pipeline_name} must be an object" + ) + for metric_name in ( + "elapsedTimeSeconds", + "totalProcessorTimeSeconds", + ): + metric = metrics.get(metric_name) + if ( + not isinstance(metric, (int, float)) + or isinstance(metric, bool) + or not math.isfinite(metric) + or metric <= 0 + ): + raise ValueError( + f"Baseline metric {pipeline_name}.{metric_name} is invalid" + ) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as error: + message = f"Ignoring invalid baseline {path}: {error}" + escaped = ( + message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + ) + print(f"::warning title=Invalid performance baseline::{escaped}") + return [] + + return runs + + +def metric_baseline( + runs: list[dict], pipeline_name: str, metric_name: str, processor_count: int +) -> list[float]: + values = [] + for run in runs: + if run.get("processorCount") != processor_count: + continue + + pipeline = run.get("pipelines", {}).get(pipeline_name) + if isinstance(pipeline, dict) and metric_name in pipeline: + values.append(float(pipeline[metric_name])) + + return values + + +def format_seconds(value: float | None) -> str: + return "n/a" if value is None else f"{value:.3f}s" + + +def format_change(value: float | None) -> str: + return "n/a" if value is None else f"{value:+.1f}%" + + +def build_summary( + pipelines: dict[str, dict[str, float]], + runs: list[dict], + processor_count: int, + threshold_percent: float, + minimum_baseline_runs: int, +) -> tuple[str, list[str]]: + lines = [ + "## PlainProcess performance regression report", + "", + f"Rolling baseline threshold: **>{threshold_percent:g}%**; " + f"runner processor count: **{processor_count}**.", + "", + "| Pipeline | Wall clock | Baseline | Change | CPU time | Baseline | Change | Status |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ] + regressions = [] + + for pipeline_name in sorted(pipelines): + current = pipelines[pipeline_name] + elapsed_values = metric_baseline( + runs, pipeline_name, "elapsedTimeSeconds", processor_count + ) + processor_values = metric_baseline( + runs, pipeline_name, "totalProcessorTimeSeconds", processor_count + ) + baseline_count = min(len(elapsed_values), len(processor_values)) + + if baseline_count < minimum_baseline_runs: + elapsed_baseline = None + processor_baseline = None + elapsed_change = None + processor_change = None + status = f"Collecting baseline ({baseline_count}/{minimum_baseline_runs})" + else: + elapsed_baseline = statistics.median(elapsed_values) + processor_baseline = statistics.median(processor_values) + elapsed_change = ( + current["elapsedTimeSeconds"] / elapsed_baseline - 1 + ) * 100 + processor_change = ( + current["totalProcessorTimeSeconds"] / processor_baseline - 1 + ) * 100 + regressed_metrics = [] + if elapsed_change > threshold_percent: + regressed_metrics.append(f"wall clock {elapsed_change:+.1f}%") + if processor_change > threshold_percent: + regressed_metrics.append(f"CPU time {processor_change:+.1f}%") + + if regressed_metrics: + status = "Regression" + regressions.append(f"{pipeline_name}: {', '.join(regressed_metrics)}") + else: + status = "Within threshold" + + lines.append( + f"| `{pipeline_name}` " + f"| {format_seconds(current['elapsedTimeSeconds'])} " + f"| {format_seconds(elapsed_baseline)} " + f"| {format_change(elapsed_change)} " + f"| {format_seconds(current['totalProcessorTimeSeconds'])} " + f"| {format_seconds(processor_baseline)} " + f"| {format_change(processor_change)} " + f"| {status} |" + ) + + if regressions: + lines.extend(["", "### Regressions", ""]) + lines.extend(f"- {regression}" for regression in regressions) + + lines.append("") + return "\n".join(lines), regressions + + +def write_updated_baseline( + output_path: Path, + prior_runs: list[dict], + run_id: str, + created_at: str, + processor_count: int, + pipelines: dict[str, dict[str, float]], + window_size: int, +) -> None: + runs = [run for run in prior_runs if str(run.get("runId")) != run_id] + runs.append( + { + "runId": run_id, + "createdAt": created_at, + "processorCount": processor_count, + "pipelines": pipelines, + } + ) + baseline = { + "schemaVersion": SCHEMA_VERSION, + "windowSize": window_size, + "runs": runs[-window_size:], + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--current-dir", type=Path, required=True) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument( + "--created-at", + default=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + ) + parser.add_argument("--threshold-percent", type=float, default=10.0) + parser.add_argument("--window-size", type=int, default=7) + parser.add_argument("--minimum-baseline-runs", type=int, default=3) + args = parser.parse_args() + + if args.window_size < args.minimum_baseline_runs: + parser.error("--window-size must be at least --minimum-baseline-runs") + + pipelines, processor_count = load_current_results(args.current_dir) + prior_runs = [ + run + for run in load_baseline(args.baseline) + if str(run.get("runId")) != args.run_id + ] + summary, regressions = build_summary( + pipelines, + prior_runs, + processor_count, + args.threshold_percent, + args.minimum_baseline_runs, + ) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(summary, encoding="utf-8") + write_updated_baseline( + args.output, + prior_runs, + args.run_id, + args.created_at, + processor_count, + pipelines, + args.window_size, + ) + + for regression in regressions: + escaped = regression.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + print(f"::warning title=Performance regression::{escaped}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/perf-timing-nightly.yml b/.github/workflows/perf-timing-nightly.yml index 5608d1d1a1..63358d22a5 100644 --- a/.github/workflows/perf-timing-nightly.yml +++ b/.github/workflows/perf-timing-nightly.yml @@ -1,9 +1,9 @@ name: Perf timing nightly -# Artifact-only perf timing collection for microsoft/testfx#9312. -# This workflow does not gate PRs and only uploads PlainProcess Result.json -# files for trend tracking. Runs on both Windows and Linux so results can be -# compared across platforms (Linux runners tend to have more consistent timing). +# Perf timing collection and non-blocking regression detection for +# microsoft/testfx#9312 and microsoft/testfx#10549. Runs on both Windows and +# Linux, but uses Linux results for the rolling baseline because hosted Linux +# runners tend to have more consistent timing. on: schedule: @@ -12,6 +12,7 @@ on: workflow_dispatch: permissions: + actions: read contents: read concurrency: @@ -123,3 +124,80 @@ jobs: path: artifacts/perf-results/ if-no-files-found: error retention-days: 90 + + - name: Download previous rolling baseline + id: previous-baseline + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + BASELINE_ZIP: ${{ runner.temp }}/perf-baseline.zip + with: + script: | + const fs = require('fs'); + const owner = context.repo.owner; + const repo = context.repo.repo; + const repository = await github.rest.repos.get({ owner, repo }); + const workflowRuns = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: 'perf-timing-nightly.yml', + status: 'completed', + branch: repository.data.default_branch, + per_page: 100, + }); + + for (const run of workflowRuns.data.workflow_runs) { + if (run.id === context.runId) { + continue; + } + + const workflowArtifacts = + await github.rest.actions.listWorkflowRunArtifacts({ + owner, + repo, + run_id: run.id, + per_page: 100, + }); + const baseline = workflowArtifacts.data.artifacts.find( + artifact => artifact.name === 'perf-timing-baseline-linux' && !artifact.expired); + if (!baseline) { + continue; + } + + const download = await github.rest.actions.downloadArtifact({ + owner, + repo, + artifact_id: baseline.id, + archive_format: 'zip', + }); + fs.writeFileSync(process.env.BASELINE_ZIP, Buffer.from(download.data)); + core.info(`Using baseline artifact ${baseline.id} from run ${run.id}.`); + core.setOutput('found', 'true'); + return; + } + + core.info('No previous rolling baseline was found; this run will initialize it.'); + core.setOutput('found', 'false'); + + - name: Extract previous rolling baseline + if: steps.previous-baseline.outputs.found == 'true' + run: | + mkdir -p "$RUNNER_TEMP/perf-baseline" + unzip -q "$RUNNER_TEMP/perf-baseline.zip" -d "$RUNNER_TEMP/perf-baseline" + + - name: Compare with rolling baseline + run: | + python .github/scripts/compare_perf_timings.py \ + --current-dir artifacts/perf-results \ + --baseline "$RUNNER_TEMP/perf-baseline/Baseline.json" \ + --output artifacts/perf-baseline/Baseline.json \ + --summary artifacts/perf-baseline/Summary.md \ + --run-id "$GITHUB_RUN_ID" + cat artifacts/perf-baseline/Summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload rolling baseline + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: perf-timing-baseline-linux + path: artifacts/perf-baseline/ + if-no-files-found: error + retention-days: 90 diff --git a/test/Performance/MSTest.Performance.Runner/Scenarios/Scenario4.cs b/test/Performance/MSTest.Performance.Runner/Scenarios/Scenario4.cs index 6763c816ba..f0f37662bf 100644 --- a/test/Performance/MSTest.Performance.Runner/Scenarios/Scenario4.cs +++ b/test/Performance/MSTest.Performance.Runner/Scenarios/Scenario4.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.TestInfrastructure; @@ -93,6 +93,7 @@ public static void ClassInitialize(TestContext context) [System.Runtime.CompilerServices.MethodImplAttribute(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public static void ClassCleanup() { + _ = _classState; _classState = 0; } """); diff --git a/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs b/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs index 0993593b64..cd06b2f056 100644 --- a/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs +++ b/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs @@ -120,9 +120,15 @@ public async Task ExecuteAsync(BuildArtifact payload, IContext context) results.Add(result); } - await File.AppendAllTextAsync( + var report = new + { + PipelineName = (string)context.Properties["PipelineName"], + Measurements = results, + }; + + await File.WriteAllTextAsync( Path.Combine(Path.GetDirectoryName(payload.TestHost.FullName)!, "Result.json"), - JsonSerializer.Serialize(results, JsonOptions)); + JsonSerializer.Serialize(report, JsonOptions)); string sample = Path.Combine(Path.GetTempPath(), _reportFileName); File.Delete(sample); diff --git a/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs b/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs index 90d0a8c3af..a9ec759233 100644 --- a/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs +++ b/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs @@ -55,9 +55,15 @@ public async Task ExecuteAsync(BuildArtifact payload, IContext context) results.Add(result); } - await File.AppendAllTextAsync(Path.Combine(Path.GetDirectoryName(payload.TestHost.FullName)!, "Result.json"), JsonSerializer.Serialize( - results, - JsonOptions)); + var report = new + { + PipelineName = (string)context.Properties["PipelineName"], + Measurements = results, + }; + + await File.WriteAllTextAsync( + Path.Combine(Path.GetDirectoryName(payload.TestHost.FullName)!, "Result.json"), + JsonSerializer.Serialize(report, JsonOptions)); string sample = Path.Combine(Path.GetTempPath(), _reportFileName); File.Delete(sample);