From 16207232aa2af6d60d849b979538e1834b26b36c Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:59:15 +0000 Subject: [PATCH 01/12] Add Windows hang dump analysis for #7554 --- .github/workflows/aot.yml | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 51e68059c5b..782ef90c12b 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -16,6 +16,11 @@ on: push: branches: ["main"] workflow_dispatch: + inputs: + analyze_windows_dump_run: + description: 'Also analyze the Windows engine hang dump from this AOT run ID (optional)' + required: false + type: string permissions: contents: read @@ -204,3 +209,42 @@ jobs: with: name: HangDump-engine-tests-${{ matrix.os }} path: '**/hangdump*' + + analyze-windows-dump: + if: github.event_name == 'workflow_dispatch' && inputs.analyze_windows_dump_run != '' + runs-on: windows-latest + timeout-minutes: 15 + permissions: + actions: read + contents: read + steps: + - uses: actions/setup-dotnet@v6.1.0 + with: + dotnet-version: '10.0.x' + + - name: Download captured Windows engine dump + uses: actions/download-artifact@v5 + with: + name: HangDump-engine-tests-windows-latest + run-id: ${{ inputs.analyze_windows_dump_run }} + github-token: ${{ github.token }} + path: captured-dump + + - name: Extract managed stacks and async waits + shell: pwsh + run: | + dotnet tool install --global dotnet-dump --version 10.0.750501 + if ($LASTEXITCODE -ne 0) { throw 'dotnet-dump installation failed' } + $dumps = @(Get-ChildItem captured-dump -Recurse -Filter '*.dmp') + if ($dumps.Count -ne 1) { throw "Expected one dump, found $($dumps.Count)" } + New-Item -ItemType Directory -Path analysis | Out-Null + dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'clrstack -all' -c 'dumpasync -stacks' -c 'exit' *> analysis/managed-stacks.txt + if ($LASTEXITCODE -ne 0) { throw 'Dump analysis failed; see managed-stacks.txt' } + + - name: Upload managed dump analysis + if: always() + uses: actions/upload-artifact@v4 + with: + name: Windows-engine-dump-analysis + path: analysis/ + if-no-files-found: warn From 03c5d0f72f47d7d15fa4ef3c8cdef099bc6db343 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:10:58 +0000 Subject: [PATCH 02/12] Fix Windows dump async analysis for #7554 --- .github/workflows/aot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 782ef90c12b..6d8b51fc73a 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -235,10 +235,10 @@ jobs: run: | dotnet tool install --global dotnet-dump --version 10.0.750501 if ($LASTEXITCODE -ne 0) { throw 'dotnet-dump installation failed' } - $dumps = @(Get-ChildItem captured-dump -Recurse -Filter '*.dmp') - if ($dumps.Count -ne 1) { throw "Expected one dump, found $($dumps.Count)" } + $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') + if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } New-Item -ItemType Directory -Path analysis | Out-Null - dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'clrstack -all' -c 'dumpasync -stacks' -c 'exit' *> analysis/managed-stacks.txt + dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt if ($LASTEXITCODE -ne 0) { throw 'Dump analysis failed; see managed-stacks.txt' } - name: Upload managed dump analysis From 5d49c63a92079eb4dfad7b7b80d570591ca782e4 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:14:36 +0000 Subject: [PATCH 03/12] Select a specific diagnostic artifact for #7554 --- .github/workflows/aot.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 6d8b51fc73a..053969c24aa 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -21,6 +21,10 @@ on: description: 'Also analyze the Windows engine hang dump from this AOT run ID (optional)' required: false type: string + analyze_windows_dump_artifact: + description: 'Exact dump artifact ID (optional; disambiguates artifacts from rerun attempts)' + required: false + type: string permissions: contents: read @@ -225,7 +229,8 @@ jobs: - name: Download captured Windows engine dump uses: actions/download-artifact@v5 with: - name: HangDump-engine-tests-windows-latest + name: ${{ inputs.analyze_windows_dump_artifact == '' && 'HangDump-engine-tests-windows-latest' || '' }} + artifact-ids: ${{ inputs.analyze_windows_dump_artifact }} run-id: ${{ inputs.analyze_windows_dump_run }} github-token: ${{ github.token }} path: captured-dump From b4dc614198dcf3b167f23dfc19f01bd677406393 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:28:40 +0000 Subject: [PATCH 04/12] Fix exact dump artifact download for #7554 --- .github/workflows/aot.yml | 41 +++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 053969c24aa..78159a3526e 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -226,11 +226,40 @@ jobs: with: dotnet-version: '10.0.x' - - name: Download captured Windows engine dump - uses: actions/download-artifact@v5 + # download-artifact filters out earlier attempts, even when given an exact ID. + - name: Download exact captured Windows engine dump + if: inputs.analyze_windows_dump_artifact != '' + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + DUMP_RUN_ID: ${{ inputs.analyze_windows_dump_run }} + DUMP_ARTIFACT_ID: ${{ inputs.analyze_windows_dump_artifact }} + run: | + New-Item -ItemType Directory -Path analysis -Force | Out-Null + try { + if ($env:DUMP_RUN_ID -notmatch '^\d+$' -or $env:DUMP_ARTIFACT_ID -notmatch '^\d+$') { + throw 'Run and artifact IDs must be numeric' + } + $artifactJson = gh api "repos/$env:GITHUB_REPOSITORY/actions/artifacts/$env:DUMP_ARTIFACT_ID" + if ($LASTEXITCODE -ne 0) { throw 'Could not read dump artifact metadata' } + $artifact = $artifactJson | ConvertFrom-Json + if ([string]$artifact.workflow_run.id -ne $env:DUMP_RUN_ID) { + throw 'The selected artifact does not belong to the requested run' + } + if ($artifact.expired) { throw 'The selected dump artifact has expired' } + $headers = @{ Authorization = "Bearer $env:GH_TOKEN"; Accept = 'application/vnd.github+json' } + Invoke-WebRequest -Uri "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions/artifacts/$env:DUMP_ARTIFACT_ID/zip" -Headers $headers -OutFile captured-dump.zip + Expand-Archive captured-dump.zip -DestinationPath captured-dump + } catch { + $_ | Out-String | Set-Content analysis/download-error.txt + throw + } + + - name: Download latest captured Windows engine dump + if: inputs.analyze_windows_dump_artifact == '' + uses: actions/download-artifact@v8.0.2 with: - name: ${{ inputs.analyze_windows_dump_artifact == '' && 'HangDump-engine-tests-windows-latest' || '' }} - artifact-ids: ${{ inputs.analyze_windows_dump_artifact }} + name: HangDump-engine-tests-windows-latest run-id: ${{ inputs.analyze_windows_dump_run }} github-token: ${{ github.token }} path: captured-dump @@ -242,13 +271,13 @@ jobs: if ($LASTEXITCODE -ne 0) { throw 'dotnet-dump installation failed' } $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } - New-Item -ItemType Directory -Path analysis | Out-Null + New-Item -ItemType Directory -Path analysis -Force | Out-Null dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt if ($LASTEXITCODE -ne 0) { throw 'Dump analysis failed; see managed-stacks.txt' } - name: Upload managed dump analysis if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7.0.2 with: name: Windows-engine-dump-analysis path: analysis/ From f27cdb01285cee5a9911d9e26f9d2407e362ca42 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:08:14 +0000 Subject: [PATCH 05/12] Capture native thread waits to investigate #7554 --- .github/workflows/aot.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 78159a3526e..99e0f941dfe 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -272,9 +272,33 @@ jobs: $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } New-Item -ItemType Directory -Path analysis -Force | Out-Null - dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt + dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'threadpool' -c 'syncblk' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt if ($LASTEXITCODE -ne 0) { throw 'Dump analysis failed; see managed-stacks.txt' } + # dotnet-dump cannot show the native wait inside Thread.StartInternal (#7554). + - name: Extract native thread waits + timeout-minutes: 5 + shell: pwsh + run: | + New-Item -ItemType Directory -Path analysis -Force | Out-Null + try { + $cdb = "${env:ProgramFiles(x86)}\Windows Kits\10\Debuggers\x64\cdb.exe" + if (!(Test-Path $cdb)) { + # Windows SDK 10.0.26100.9457; install only its debugging tools. + Invoke-WebRequest 'https://go.microsoft.com/fwlink/?linkid=2382321' -OutFile winsdksetup.exe + $installer = Start-Process -FilePath .\winsdksetup.exe -ArgumentList '/features OptionId.WindowsDesktopDebuggers /quiet /norestart' -Wait -PassThru + if ($installer.ExitCode -notin @(0, 3010)) { throw "Debugger installation failed: $($installer.ExitCode)" } + } + if (!(Test-Path $cdb)) { throw 'Windows SDK did not install x64 cdb.exe' } + $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') + if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } + & $cdb -z $dumps[0].FullName -y 'srv*symbols*https://msdl.microsoft.com/download/symbols' -c '.reload; ~* kb; !locks; q' *> analysis/native-stacks.txt + if ($LASTEXITCODE -ne 0) { throw 'Native dump analysis failed; see native-stacks.txt' } + } catch { + $_ | Out-String | Set-Content analysis/native-analysis-error.txt + throw + } + - name: Upload managed dump analysis if: always() uses: actions/upload-artifact@v7.0.2 From 1efd64e9b65688ec07d0832a7bd773f2e7e25345 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:21:02 +0000 Subject: [PATCH 06/12] Fix native dump symbol cache path for #7554 --- .github/workflows/aot.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 99e0f941dfe..0b524613dc4 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -292,7 +292,9 @@ jobs: if (!(Test-Path $cdb)) { throw 'Windows SDK did not install x64 cdb.exe' } $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } - & $cdb -z $dumps[0].FullName -y 'srv*symbols*https://msdl.microsoft.com/download/symbols' -c '.reload; ~* kb; !locks; q' *> analysis/native-stacks.txt + $symbolCache = Join-Path $env:RUNNER_TEMP 'engine-dump-symbols' + New-Item -ItemType Directory -Path $symbolCache -Force | Out-Null + & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c '.reload; ~* kb; !locks; q' *> analysis/native-stacks.txt if ($LASTEXITCODE -ne 0) { throw 'Native dump analysis failed; see native-stacks.txt' } } catch { $_ | Out-String | Set-Content analysis/native-analysis-error.txt From d095e46be917087aa35b6d82326a393ad9ce5001 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:23:28 +0000 Subject: [PATCH 07/12] Preserve independent dump analysis after failures (#7554) --- .github/workflows/aot.yml | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 0b524613dc4..9c5a25087bd 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -256,6 +256,7 @@ jobs: } - name: Download latest captured Windows engine dump + id: download_name_based if: inputs.analyze_windows_dump_artifact == '' uses: actions/download-artifact@v8.0.2 with: @@ -264,19 +265,37 @@ jobs: github-token: ${{ github.token }} path: captured-dump + - name: Record name-based dump download failure + if: ${{ always() && steps.download_name_based.outcome == 'failure' }} + shell: pwsh + env: + DUMP_RUN_ID: ${{ inputs.analyze_windows_dump_run }} + run: | + New-Item -ItemType Directory -Path analysis -Force | Out-Null + "Name-based Windows dump download failed for run $env:DUMP_RUN_ID" | Set-Content analysis/download-error.txt + throw 'Name-based Windows dump download failed' + - name: Extract managed stacks and async waits + if: ${{ !cancelled() }} + timeout-minutes: 5 shell: pwsh run: | - dotnet tool install --global dotnet-dump --version 10.0.750501 - if ($LASTEXITCODE -ne 0) { throw 'dotnet-dump installation failed' } - $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') - if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } New-Item -ItemType Directory -Path analysis -Force | Out-Null - dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'threadpool' -c 'syncblk' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt - if ($LASTEXITCODE -ne 0) { throw 'Dump analysis failed; see managed-stacks.txt' } + try { + dotnet tool install --global dotnet-dump --version 10.0.750501 + if ($LASTEXITCODE -ne 0) { throw 'dotnet-dump installation failed' } + $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') + if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } + dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'threadpool' -c 'syncblk' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt + if ($LASTEXITCODE -ne 0) { throw 'Dump analysis failed; see managed-stacks.txt' } + } catch { + $_ | Out-String | Set-Content analysis/managed-analysis-error.txt + throw + } # dotnet-dump cannot show the native wait inside Thread.StartInternal (#7554). - name: Extract native thread waits + if: ${{ !cancelled() }} timeout-minutes: 5 shell: pwsh run: | From 9dab6a47c83990987926b60a05f7e120541e38fa Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:41:35 +0000 Subject: [PATCH 08/12] Inspect thread-start target state and preserve analysis failures (#7554) --- .github/workflows/aot.yml | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 9c5a25087bd..6435c764ede 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -228,6 +228,7 @@ jobs: # download-artifact filters out earlier attempts, even when given an exact ID. - name: Download exact captured Windows engine dump + id: download_exact if: inputs.analyze_windows_dump_artifact != '' shell: pwsh env: @@ -276,7 +277,8 @@ jobs: throw 'Name-based Windows dump download failed' - name: Extract managed stacks and async waits - if: ${{ !cancelled() }} + id: managed_analysis + if: ${{ !cancelled() && (steps.download_exact.outcome == 'success' || steps.download_name_based.outcome == 'success') }} timeout-minutes: 5 shell: pwsh run: | @@ -286,16 +288,35 @@ jobs: if ($LASTEXITCODE -ne 0) { throw 'dotnet-dump installation failed' } $dumps = @(Get-ChildItem captured-dump -Recurse -Filter 'hangdump.windows-reflection-pass.dmp') if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } - dotnet-dump analyze $dumps[0].FullName -c 'clrthreads' -c 'threadpool' -c 'syncblk' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt + dotnet-dump analyze $dumps[0].FullName -c 'eeversion' -c 'clrthreads' -c 'threadpool' -c 'syncblk' -c 'clrstack -all' -c 'dumpasync' -c 'exit' *> analysis/managed-stacks.txt if ($LASTEXITCODE -ne 0) { throw 'Dump analysis failed; see managed-stacks.txt' } + # Map Thread object monitors to their native thread pointers and IDs (#7554). + $threadCommands = @('-c', 'eeversion') + foreach ($line in Get-Content analysis/managed-stacks.txt) { + if ($line.Trim() -match '([0-9a-fA-F]{16}) System\.Threading\.Thread$') { + $threadCommands += @('-c', "dumpobj $($Matches[1])") + } + } + $threadCommands += @('-c', 'exit') + dotnet-dump analyze $dumps[0].FullName @threadCommands *> analysis/thread-objects.txt + if ($LASTEXITCODE -ne 0) { throw 'Thread object analysis failed; see thread-objects.txt' } } catch { $_ | Out-String | Set-Content analysis/managed-analysis-error.txt throw } + - name: Record interrupted managed analysis + if: ${{ !cancelled() && steps.managed_analysis.outcome == 'failure' }} + shell: pwsh + run: | + New-Item -ItemType Directory -Path analysis -Force | Out-Null + if (!(Test-Path analysis/managed-analysis-error.txt)) { + 'Managed analysis failed or timed out before its error handler completed; inspect any partial output.' | Set-Content analysis/managed-analysis-error.txt + } + # dotnet-dump cannot show the native wait inside Thread.StartInternal (#7554). - name: Extract native thread waits - if: ${{ !cancelled() }} + if: ${{ !cancelled() && (steps.download_exact.outcome == 'success' || steps.download_name_based.outcome == 'success') }} timeout-minutes: 5 shell: pwsh run: | @@ -313,7 +334,7 @@ jobs: if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } $symbolCache = Join-Path $env:RUNNER_TEMP 'engine-dump-symbols' New-Item -ItemType Directory -Path $symbolCache -Force | Out-Null - & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c '.reload; ~* kb; !locks; q' *> analysis/native-stacks.txt + & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c '.reload; lmvm coreclr; uf coreclr!ThreadNative_Start; ~* r; ~* kb; !locks; q' *> analysis/native-stacks.txt if ($LASTEXITCODE -ne 0) { throw 'Native dump analysis failed; see native-stacks.txt' } } catch { $_ | Out-String | Set-Content analysis/native-analysis-error.txt From 568508f3db660d5afe23613ec850638cb45937b6 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:48:57 +0000 Subject: [PATCH 09/12] Record interrupted native dump analysis for #7554 --- .github/workflows/aot.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 6435c764ede..a92e93a4271 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -316,6 +316,7 @@ jobs: # dotnet-dump cannot show the native wait inside Thread.StartInternal (#7554). - name: Extract native thread waits + id: native_analysis if: ${{ !cancelled() && (steps.download_exact.outcome == 'success' || steps.download_name_based.outcome == 'success') }} timeout-minutes: 5 shell: pwsh @@ -341,6 +342,15 @@ jobs: throw } + - name: Record interrupted native analysis + if: ${{ always() && steps.native_analysis.outcome == 'failure' }} + shell: pwsh + run: | + New-Item -ItemType Directory -Path analysis -Force | Out-Null + if (!(Test-Path analysis/native-analysis-error.txt)) { + 'Native analysis failed or timed out before its error handler completed; inspect any partial output.' | Set-Content analysis/native-analysis-error.txt + } + - name: Upload managed dump analysis if: always() uses: actions/upload-artifact@v7.0.2 From 0e007a0831fb6c51f7a2bf628bf14d225ffef25b Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:35:49 +0000 Subject: [PATCH 10/12] Capture unwound thread-start targets for #7554 --- .github/workflows/aot.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index a92e93a4271..45efd0a93ab 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -335,8 +335,24 @@ jobs: if ($dumps.Count -ne 1) { throw "Expected one Windows reflection pass dump, found $($dumps.Count)" } $symbolCache = Join-Path $env:RUNNER_TEMP 'engine-dump-symbols' New-Item -ItemType Directory -Path $symbolCache -Force | Out-Null - & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c '.reload; lmvm coreclr; uf coreclr!ThreadNative_Start; ~* r; ~* kb; !locks; q' *> analysis/native-stacks.txt + & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c '.reload; lmvm coreclr; uf coreclr!ThreadNative_Start; ~* r; ~* kn; !locks; q' *> analysis/native-stacks.txt if ($LASTEXITCODE -ne 0) { throw 'Native dump analysis failed; see native-stacks.txt' } + # Top-frame registers belong to SleepEx/SwitchToThread, not ThreadNative_Start. + # Unwind each starter before reading its preserved RBX (the target native Thread). + $targetCommands = @('.reload') + $threadIndex = $null + foreach ($line in Get-Content analysis/native-stacks.txt) { + if ($line -match '^\s*(\d+)\s+Id:') { + $threadIndex = $Matches[1] + } elseif ($null -ne $threadIndex -and $line -match '^\s*([0-9a-fA-F]+)\s+.*coreclr!ThreadNative_Start\+') { + $frameIndex = $Matches[1] + $targetCommands += "~$($threadIndex)s; .frame /r 0x$frameIndex; dd @rbx L4; dq @rbx+0x110 L1" + } + } + if ($targetCommands.Count -eq 1) { throw 'No numbered ThreadNative_Start frames found; see native-stacks.txt' } + $targetCommands += 'q' + & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c ($targetCommands -join '; ') *> analysis/start-targets.txt + if ($LASTEXITCODE -ne 0) { throw 'Starter target analysis failed; see start-targets.txt' } } catch { $_ | Out-String | Set-Content analysis/native-analysis-error.txt throw From 2f514b301d8982336982daeb3963ab663800a652 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:39:18 +0000 Subject: [PATCH 11/12] Read explicit unwound thread target addresses for #7554 --- .github/workflows/aot.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 45efd0a93ab..0a87d10faa7 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -346,13 +346,26 @@ jobs: $threadIndex = $Matches[1] } elseif ($null -ne $threadIndex -and $line -match '^\s*([0-9a-fA-F]+)\s+.*coreclr!ThreadNative_Start\+') { $frameIndex = $Matches[1] - $targetCommands += "~$($threadIndex)s; .frame /r 0x$frameIndex; dd @rbx L4; dq @rbx+0x110 L1" + $targetCommands += "~$($threadIndex)s; .frame /r 0x$frameIndex" } } if ($targetCommands.Count -eq 1) { throw 'No numbered ThreadNative_Start frames found; see native-stacks.txt' } $targetCommands += 'q' & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c ($targetCommands -join '; ') *> analysis/start-targets.txt if ($LASTEXITCODE -ne 0) { throw 'Starter target analysis failed; see start-targets.txt' } + # .frame /r prints unwound registers, but @rbx still evaluates in the top frame. + # Read explicit addresses from that output rather than debugger register expressions. + $targetAddresses = @(Get-Content analysis/start-targets.txt | ForEach-Object { + if ($_ -match '^rax=[0-9a-fA-F]+\s+rbx=([0-9a-fA-F]+)\s+rcx=') { $Matches[1] } + } | Select-Object -Unique) + if ($targetAddresses.Count -eq 0) { throw 'No unwound RBX addresses found; see start-targets.txt' } + $stateCommands = @('.reload') + foreach ($address in $targetAddresses) { + $stateCommands += "dd 0x$address L4; dq 0x$address+0x110 L1" + } + $stateCommands += 'q' + & $cdb -z $dumps[0].FullName -y "srv*$symbolCache*https://msdl.microsoft.com/download/symbols" -c ($stateCommands -join '; ') *> analysis/start-target-states.txt + if ($LASTEXITCODE -ne 0) { throw 'Starter state analysis failed; see start-target-states.txt' } } catch { $_ | Out-String | Set-Content analysis/native-analysis-error.txt throw From 02ab3932325334d6e624ee354e75b8c05feb283a Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:39:43 +0000 Subject: [PATCH 12/12] Accept CDB current and event thread markers for #7554 --- .github/workflows/aot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 0a87d10faa7..6e47a22abf1 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -342,7 +342,7 @@ jobs: $targetCommands = @('.reload') $threadIndex = $null foreach ($line in Get-Content analysis/native-stacks.txt) { - if ($line -match '^\s*(\d+)\s+Id:') { + if ($line -match '^\s*[.#]?\s*(\d+)\s+Id:') { $threadIndex = $Matches[1] } elseif ($null -ne $threadIndex -and $line -match '^\s*([0-9a-fA-F]+)\s+.*coreclr!ThreadNative_Start\+') { $frameIndex = $Matches[1]