From d179ca1f7b2f7f934ea032ba83508152852dccdf Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:03:20 +0000 Subject: [PATCH 1/3] fix: initialize inbox test relational model before publication (#570) --- .../InboxTests.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/Relayer.Outbox.PostgreSql.Tests/InboxTests.cs b/tests/Relayer.Outbox.PostgreSql.Tests/InboxTests.cs index 00ad4759..aa3f3a27 100644 --- a/tests/Relayer.Outbox.PostgreSql.Tests/InboxTests.cs +++ b/tests/Relayer.Outbox.PostgreSql.Tests/InboxTests.cs @@ -3,6 +3,8 @@ using System.Data; using System.Data.Common; using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Conventions; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; @@ -17,6 +19,16 @@ namespace Relayer.Outbox.PostgreSql.Tests; [InheritsTests] public class InboxTests : InboxContractTests { + [Test] + public async Task Orders_model_is_ready_before_concurrent_database_work() + { + var options = new DbContextOptionsBuilder().UseNpgsql("Host=unused").Options; + await using var context = new OrdersContext(options, new Control()); + // Reading Model constructs the native runtime model without issuing a database operation. + // EF's first-use race is reachable if the relational model is still lazy at this point. + await Assert.That(context.Model.FindRuntimeAnnotationValue(RelationalAnnotationNames.RelationalModel)).IsNotNull(); + } + [Test] [Arguments(false, false, 2)] [Arguments(false, true, 1)] @@ -551,6 +563,9 @@ public async ValueTask HandleAsync(Probe message, MessageContext context, Cancel public sealed class OrdersContext(DbContextOptions options, Control control) : DbContext(options) { + protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) => + configurationBuilder.Conventions.Add(static _ => new InitializeRelationalModel()); + protected override void OnModelCreating(ModelBuilder modelBuilder) => modelBuilder.Entity().ToTable("orders").HasKey(x => x.Id); public override async ValueTask DisposeAsync() { @@ -558,6 +573,18 @@ public override async ValueTask DisposeAsync() control.OnDispose?.Invoke(); } } + // EF 10.0.15 has a first-use relational-model race (dotnet/efcore#39214). + // Build it in EF's existing model-construction pipeline before publishing the cached + // runtime model. Tests still execute concurrently; remove after upgrading to 10.0.16. + private sealed class InitializeRelationalModel : IModelFinalizedConvention + { + public IModel ProcessModelFinalized(IModel model) + { + _ = model.GetRelationalModel(); + return model; + } + } + public sealed class OtherContext(DbContextOptions options) : DbContext(options); public sealed class Order { public Guid Id { get; set; } } [Message("inbox.probe")] From b3b0e29459449502154f3da22edf14d9b3948066 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:32:28 +0000 Subject: [PATCH 2/3] Investigate #571 batch allocation failure with exact hosted callbacks --- .github/scripts/batch_allocation_control.py | 39 ++++++++++ .../workflows/batch-allocation-control.yml | 75 +++++++++++++++++++ tests/Relayer.Tests/BatchPipelineTests.cs | 9 +++ tests/Relayer.Tests/Relayer.Tests.csproj | 1 + 4 files changed, 124 insertions(+) create mode 100644 .github/scripts/batch_allocation_control.py create mode 100644 .github/workflows/batch-allocation-control.yml diff --git a/.github/scripts/batch_allocation_control.py b/.github/scripts/batch_allocation_control.py new file mode 100644 index 00000000..b57a2695 --- /dev/null +++ b/.github/scripts/batch_allocation_control.py @@ -0,0 +1,39 @@ +"""Fixed-count Linux diagnostic for #571; never retries or replaces a CI gate.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +from read_native_allocations import read + +image, profiler, destination = map(lambda value: Path(value).resolve(), sys.argv[1:]) +destination.mkdir(parents=True, exist_ok=True) +results = [] +# Every outcome is retained; a passing trial never discards an earlier failure. +for trial in range(8): + for instrumented in (False, True): + name = f'{trial:02d}-' + ('callbacks' if instrumented else 'original') + report = destination / f'{name}.jsonl' + environment = dict(os.environ) + for key in ("CORECLR_ENABLE_PROFILING", "CORECLR_PROFILER", "CORECLR_PROFILER_PATH", + "RELAYER_ALLOCATION_PROFILER", "RELAYER_ALLOCATION_OUTPUT"): + environment.pop(key, None) + if instrumented: + environment.update(CORECLR_ENABLE_PROFILING='1', + CORECLR_PROFILER='{32B7EF07-25E8-4AC7-B011-9A4321D702C8}', + CORECLR_PROFILER_PATH=str(profiler), + RELAYER_ALLOCATION_PROFILER=str(profiler), + RELAYER_ALLOCATION_OUTPUT=str(report)) + outcome = subprocess.run(['dotnet', str(image), '--treenode-filter', + '/*/*/BatchPipelineTests/Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_scope_and_sender'], + env=environment, capture_output=True, text=True, timeout=120) + (destination / f'{name}.log').write_text(outcome.stdout + outcome.stderr) + row = {'trial': trial, 'instrumented': instrumented, 'exit_code': outcome.returncode} + if instrumented: + row['allocation_report'] = read(report) + results.append(row) + (destination / 'outcomes.json').write_text(json.dumps(results, indent=2) + '\n') + print(name, 'test exit', outcome.returncode, flush=True) +# Test failures are diagnostic observations, not suppressed CI failures. The original +# ready-PR CI check remains failed. Invalid/missing callback reports fail this job. diff --git a/.github/workflows/batch-allocation-control.yml b/.github/workflows/batch-allocation-control.yml new file mode 100644 index 00000000..d62bc50e --- /dev/null +++ b/.github/workflows/batch-allocation-control.yml @@ -0,0 +1,75 @@ +name: Batch allocation diagnostic + +# Separate instrumented investigation of #571, never a replacement CI gate. +on: + pull_request: + paths: + - '.github/workflows/batch-allocation-control.yml' + - '.github/scripts/batch_allocation_control.py' + workflow_dispatch: + +permissions: + contents: read + +jobs: + capture: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: dotnet/samples + ref: 3810558800760fc39282cc939bac645735514209 + path: profiler-sdk + sparse-checkout: core/profiling + persist-credentials: false + - uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6.1.0 + with: + dotnet-version: | + 10.0.403 + 11.0.100 + - name: Build original comparison and existing exact CLR profiler + run: | + mkdir -p results + dotnet --info > results/runtime.txt + uname -a > results/host.txt + CXX=clang++ cmake -S tools/FrozenAllocationProfiler -B .artifacts/exact-profiler -DPROFILER_SAMPLES="$PWD/profiler-sdk" -DCMAKE_BUILD_TYPE=Release > results/native-configure.log 2>&1 + cmake --build .artifacts/exact-profiler > results/native-build.log 2>&1 + dotnet build tools/FrozenAllocationProfiler/Fixture -c Release > results/fixture-build.log 2>&1 + dotnet build tests/Relayer.Tests -c Release -f net10.0 > results/test-build.log 2>&1 + - name: Verify exact counts with existing four-thread known-type fixture + env: + TEST_NATIVE_PROFILER: .artifacts/exact-profiler/libFrozenAllocationProfiler.so + TEST_NATIVE_FIXTURE: tools/FrozenAllocationProfiler/Fixture/bin/Release/net10.0/Fixture.dll + TEST_NATIVE_OUTPUT: results/fixture + run: python3 -m unittest discover -s .github/scripts -p test_native_allocations.py > results/native-tests.log 2>&1 + - name: Capture all sixteen predeclared original and instrumented trials + run: | + python3 .github/scripts/batch_allocation_control.py tests/Relayer.Tests/bin/Release/net10.0/Relayer.Tests.dll .artifacts/exact-profiler/libFrozenAllocationProfiler.so results/trials + echo 'Diagnostic only: all test exit codes are in outcomes.json. Callbacks change fast paths and scheduling; counts include all threads, have no stacks, and cannot attribute the historical +56 B window. Original CI failure is not replaced.' >> "$GITHUB_STEP_SUMMARY" + - name: Inventory diagnostic inputs and retain the installed runtime + if: always() + run: | + mkdir -p results/runtime + cp -a "$DOTNET_ROOT/shared/Microsoft.NETCore.App/10.0."* results/runtime/ + find results tests/Relayer.Tests src tools/FrozenAllocationProfiler .artifacts/exact-profiler profiler-sdk/core/profiling -type f -exec sha256sum {} + > results-SHA256SUMS + - name: Retain all outcomes, sources, loaded test binaries and profiler inputs + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: batch-allocation-${{ github.run_id }}-${{ github.run_attempt }} + path: | + results/ + results-SHA256SUMS + tests/Relayer.Tests/ + src/ + tools/FrozenConcurrentProbe/NativeAllocationRegion.cs + tools/FrozenAllocationProfiler/ + .artifacts/exact-profiler/ + profiler-sdk/core/profiling/ + .github/scripts/batch_allocation_control.py + .github/scripts/read_native_allocations.py + .github/workflows/batch-allocation-control.yml + include-hidden-files: true + retention-days: 90 diff --git a/tests/Relayer.Tests/BatchPipelineTests.cs b/tests/Relayer.Tests/BatchPipelineTests.cs index 0d42caab..36f15145 100644 --- a/tests/Relayer.Tests/BatchPipelineTests.cs +++ b/tests/Relayer.Tests/BatchPipelineTests.cs @@ -541,6 +541,8 @@ public async Task Batch_context_ownership_and_single_message_entry_are_validated [Arguments(true)] public async Task Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_scope_and_sender(bool bufferedSends) { + // Optional exact-callback capture for the separate hosted diagnostic, never normal CI. + var allocationRegion = bufferedSends ? NativeAllocationRegion.Load() : null; await using var provider = Provider(size: 10); provider.GetRequiredService().BufferedSends = bufferedSends; using var batchCancellation = new CancellationTokenSource(); @@ -566,6 +568,9 @@ public async Task Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_sc _ = scope.ServiceProvider.GetRequiredService(); } var baseline = GC.GetAllocatedBytesForCurrentThread() - start; + var thread = Environment.CurrentManagedThreadId; + var collections = (GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2)); + allocationRegion?.Begin(); start = GC.GetAllocatedBytesForCurrentThread(); for (var iteration = 0; iteration < 1000; iteration++) { @@ -574,6 +579,10 @@ public async Task Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_sc for (var i = 0; i < contexts.Length; i++) contexts[i].Dispose(); } var composed = GC.GetAllocatedBytesForCurrentThread() - start; + allocationRegion?.End(); + Console.WriteLine($"Batch allocation diagnostic: buffered={bufferedSends}, baseline={baseline}, composed={composed}, " + + $"thread={thread}->{Environment.CurrentManagedThreadId}, collections={collections}->" + + $"({GC.CollectionCount(0)}, {GC.CollectionCount(1)}, {GC.CollectionCount(2)}), runtime={Environment.Version}"); foreach (var delivery in deliveries) delivery.Dispose(); await Assert.That(composed).IsEqualTo(baseline); } diff --git a/tests/Relayer.Tests/Relayer.Tests.csproj b/tests/Relayer.Tests/Relayer.Tests.csproj index cc03077c..e3fcf7fe 100644 --- a/tests/Relayer.Tests/Relayer.Tests.csproj +++ b/tests/Relayer.Tests/Relayer.Tests.csproj @@ -40,6 +40,7 @@ + From 573afcdb8dae74a843ae9b6e04a40424aa58c789 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:37:55 +0000 Subject: [PATCH 3/3] Revert "Investigate #571 batch allocation failure with exact hosted callbacks" This reverts commit b3b0e29459449502154f3da22edf14d9b3948066. --- .github/scripts/batch_allocation_control.py | 39 ---------- .../workflows/batch-allocation-control.yml | 75 ------------------- tests/Relayer.Tests/BatchPipelineTests.cs | 9 --- tests/Relayer.Tests/Relayer.Tests.csproj | 1 - 4 files changed, 124 deletions(-) delete mode 100644 .github/scripts/batch_allocation_control.py delete mode 100644 .github/workflows/batch-allocation-control.yml diff --git a/.github/scripts/batch_allocation_control.py b/.github/scripts/batch_allocation_control.py deleted file mode 100644 index b57a2695..00000000 --- a/.github/scripts/batch_allocation_control.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Fixed-count Linux diagnostic for #571; never retries or replaces a CI gate.""" -import json -import os -from pathlib import Path -import subprocess -import sys - -from read_native_allocations import read - -image, profiler, destination = map(lambda value: Path(value).resolve(), sys.argv[1:]) -destination.mkdir(parents=True, exist_ok=True) -results = [] -# Every outcome is retained; a passing trial never discards an earlier failure. -for trial in range(8): - for instrumented in (False, True): - name = f'{trial:02d}-' + ('callbacks' if instrumented else 'original') - report = destination / f'{name}.jsonl' - environment = dict(os.environ) - for key in ("CORECLR_ENABLE_PROFILING", "CORECLR_PROFILER", "CORECLR_PROFILER_PATH", - "RELAYER_ALLOCATION_PROFILER", "RELAYER_ALLOCATION_OUTPUT"): - environment.pop(key, None) - if instrumented: - environment.update(CORECLR_ENABLE_PROFILING='1', - CORECLR_PROFILER='{32B7EF07-25E8-4AC7-B011-9A4321D702C8}', - CORECLR_PROFILER_PATH=str(profiler), - RELAYER_ALLOCATION_PROFILER=str(profiler), - RELAYER_ALLOCATION_OUTPUT=str(report)) - outcome = subprocess.run(['dotnet', str(image), '--treenode-filter', - '/*/*/BatchPipelineTests/Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_scope_and_sender'], - env=environment, capture_output=True, text=True, timeout=120) - (destination / f'{name}.log').write_text(outcome.stdout + outcome.stderr) - row = {'trial': trial, 'instrumented': instrumented, 'exit_code': outcome.returncode} - if instrumented: - row['allocation_report'] = read(report) - results.append(row) - (destination / 'outcomes.json').write_text(json.dumps(results, indent=2) + '\n') - print(name, 'test exit', outcome.returncode, flush=True) -# Test failures are diagnostic observations, not suppressed CI failures. The original -# ready-PR CI check remains failed. Invalid/missing callback reports fail this job. diff --git a/.github/workflows/batch-allocation-control.yml b/.github/workflows/batch-allocation-control.yml deleted file mode 100644 index d62bc50e..00000000 --- a/.github/workflows/batch-allocation-control.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Batch allocation diagnostic - -# Separate instrumented investigation of #571, never a replacement CI gate. -on: - pull_request: - paths: - - '.github/workflows/batch-allocation-control.yml' - - '.github/scripts/batch_allocation_control.py' - workflow_dispatch: - -permissions: - contents: read - -jobs: - capture: - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: dotnet/samples - ref: 3810558800760fc39282cc939bac645735514209 - path: profiler-sdk - sparse-checkout: core/profiling - persist-credentials: false - - uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6.1.0 - with: - dotnet-version: | - 10.0.403 - 11.0.100 - - name: Build original comparison and existing exact CLR profiler - run: | - mkdir -p results - dotnet --info > results/runtime.txt - uname -a > results/host.txt - CXX=clang++ cmake -S tools/FrozenAllocationProfiler -B .artifacts/exact-profiler -DPROFILER_SAMPLES="$PWD/profiler-sdk" -DCMAKE_BUILD_TYPE=Release > results/native-configure.log 2>&1 - cmake --build .artifacts/exact-profiler > results/native-build.log 2>&1 - dotnet build tools/FrozenAllocationProfiler/Fixture -c Release > results/fixture-build.log 2>&1 - dotnet build tests/Relayer.Tests -c Release -f net10.0 > results/test-build.log 2>&1 - - name: Verify exact counts with existing four-thread known-type fixture - env: - TEST_NATIVE_PROFILER: .artifacts/exact-profiler/libFrozenAllocationProfiler.so - TEST_NATIVE_FIXTURE: tools/FrozenAllocationProfiler/Fixture/bin/Release/net10.0/Fixture.dll - TEST_NATIVE_OUTPUT: results/fixture - run: python3 -m unittest discover -s .github/scripts -p test_native_allocations.py > results/native-tests.log 2>&1 - - name: Capture all sixteen predeclared original and instrumented trials - run: | - python3 .github/scripts/batch_allocation_control.py tests/Relayer.Tests/bin/Release/net10.0/Relayer.Tests.dll .artifacts/exact-profiler/libFrozenAllocationProfiler.so results/trials - echo 'Diagnostic only: all test exit codes are in outcomes.json. Callbacks change fast paths and scheduling; counts include all threads, have no stacks, and cannot attribute the historical +56 B window. Original CI failure is not replaced.' >> "$GITHUB_STEP_SUMMARY" - - name: Inventory diagnostic inputs and retain the installed runtime - if: always() - run: | - mkdir -p results/runtime - cp -a "$DOTNET_ROOT/shared/Microsoft.NETCore.App/10.0."* results/runtime/ - find results tests/Relayer.Tests src tools/FrozenAllocationProfiler .artifacts/exact-profiler profiler-sdk/core/profiling -type f -exec sha256sum {} + > results-SHA256SUMS - - name: Retain all outcomes, sources, loaded test binaries and profiler inputs - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: batch-allocation-${{ github.run_id }}-${{ github.run_attempt }} - path: | - results/ - results-SHA256SUMS - tests/Relayer.Tests/ - src/ - tools/FrozenConcurrentProbe/NativeAllocationRegion.cs - tools/FrozenAllocationProfiler/ - .artifacts/exact-profiler/ - profiler-sdk/core/profiling/ - .github/scripts/batch_allocation_control.py - .github/scripts/read_native_allocations.py - .github/workflows/batch-allocation-control.yml - include-hidden-files: true - retention-days: 90 diff --git a/tests/Relayer.Tests/BatchPipelineTests.cs b/tests/Relayer.Tests/BatchPipelineTests.cs index 36f15145..0d42caab 100644 --- a/tests/Relayer.Tests/BatchPipelineTests.cs +++ b/tests/Relayer.Tests/BatchPipelineTests.cs @@ -541,8 +541,6 @@ public async Task Batch_context_ownership_and_single_message_entry_are_validated [Arguments(true)] public async Task Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_scope_and_sender(bool bufferedSends) { - // Optional exact-callback capture for the separate hosted diagnostic, never normal CI. - var allocationRegion = bufferedSends ? NativeAllocationRegion.Load() : null; await using var provider = Provider(size: 10); provider.GetRequiredService().BufferedSends = bufferedSends; using var batchCancellation = new CancellationTokenSource(); @@ -568,9 +566,6 @@ public async Task Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_sc _ = scope.ServiceProvider.GetRequiredService(); } var baseline = GC.GetAllocatedBytesForCurrentThread() - start; - var thread = Environment.CurrentManagedThreadId; - var collections = (GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2)); - allocationRegion?.Begin(); start = GC.GetAllocatedBytesForCurrentThread(); for (var iteration = 0; iteration < 1000; iteration++) { @@ -579,10 +574,6 @@ public async Task Warmed_batches_add_no_allocations_beyond_identical_fresh_DI_sc for (var i = 0; i < contexts.Length; i++) contexts[i].Dispose(); } var composed = GC.GetAllocatedBytesForCurrentThread() - start; - allocationRegion?.End(); - Console.WriteLine($"Batch allocation diagnostic: buffered={bufferedSends}, baseline={baseline}, composed={composed}, " + - $"thread={thread}->{Environment.CurrentManagedThreadId}, collections={collections}->" + - $"({GC.CollectionCount(0)}, {GC.CollectionCount(1)}, {GC.CollectionCount(2)}), runtime={Environment.Version}"); foreach (var delivery in deliveries) delivery.Dispose(); await Assert.That(composed).IsEqualTo(baseline); } diff --git a/tests/Relayer.Tests/Relayer.Tests.csproj b/tests/Relayer.Tests/Relayer.Tests.csproj index e3fcf7fe..cc03077c 100644 --- a/tests/Relayer.Tests/Relayer.Tests.csproj +++ b/tests/Relayer.Tests/Relayer.Tests.csproj @@ -40,7 +40,6 @@ -