From 710e7298ffc4a0f8414a021f58cfc27d109e900f Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Tue, 18 Feb 2025 14:47:55 -0800 Subject: [PATCH 1/3] Move local thread pool work items to global by default when blocking on a task - Enables the change in https://github.com/dotnet/runtime/pull/109841 by default - A worker thread blocking on a task while having pending local work items can lead to priority inversion and deadlock-like issues, as local work items are stolen with the lowest priority by other threads. In a situation where all worker threads are blocked on tasks and new ones continue to block on tasks similarly, the global queues may not deplete for a long time and any local work items of blocked threads may not run meanwhile. There have been some reports of issues like this occurring while coinciding with other external issues that aggravate the problem. - When a worker thread blocks on a task, the change moves any local work items to the high-priority global queue. Using the normal-priority queues can lead to other priority inversion issues in worst-case scenarios like above, where the normal-priority queues may consist of a number of new requests, and local work items typically represent already in-flight work. --- .../src/System/Threading/Tasks/Task.cs | 35 ++++++++----------- .../System/Threading/ThreadPoolWorkQueue.cs | 25 +++++++++---- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs index 97b07d08bed620..d007bc756e8bdb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @@ -3060,26 +3060,21 @@ private bool SpinThenBlockingWait(int millisecondsTimeout, CancellationToken can bool returnValue = SpinWait(millisecondsTimeout); if (!returnValue) { -#if CORECLR - if (ThreadPoolWorkQueue.s_prioritizationExperiment) - { - // We're about to block waiting for the task to complete, which is expensive, and if - // the task being waited on depends on some other work to run, this thread could end up - // waiting for some other thread to do work. If the two threads are part of the same scheduler, - // such as the thread pool, that could lead to a (temporary) deadlock. This is made worse by - // it also leading to a possible priority inversion on previously queued work. Each thread in - // the thread pool has a local queue. A key motivator for this local queue is it allows this - // thread to create work items that it will then prioritize above all other work in the - // pool. However, while this thread makes its own local queue the top priority, that queue is - // every other thread's lowest priority. If this thread blocks, all of its created work that's - // supposed to be high priority becomes low priority, and work that's typically part of a - // currently in-flight operation gets deprioritized relative to new requests coming into the - // pool, which can lead to the whole system slowing down or even deadlocking. To address that, - // just before we block, we move all local work into a global queue, so that it's at least - // prioritized by other threads more fairly with respect to other work. - ThreadPoolWorkQueue.TransferAllLocalWorkItemsToHighPriorityGlobalQueue(); - } -#endif + // We're about to block waiting for the task to complete, which is expensive, and if + // the task being waited on depends on some other work to run, this thread could end up + // waiting for some other thread to do work. If the two threads are part of the same scheduler, + // such as the thread pool, that could lead to a (temporary) deadlock. This is made worse by + // it also leading to a possible priority inversion on previously queued work. Each thread in + // the thread pool has a local queue. A key motivator for this local queue is it allows this + // thread to create work items that it will then prioritize above all other work in the + // pool. However, while this thread makes its own local queue the top priority, that queue is + // every other thread's lowest priority. If this thread blocks, all of its created work that's + // supposed to be high priority becomes low priority, and work that's typically part of a + // currently in-flight operation gets deprioritized relative to new requests coming into the + // pool, which can lead to the whole system slowing down or even deadlocking. To address that, + // just before we block, we move all local work into a global queue, so that it's at least + // prioritized by other threads more fairly with respect to other work. + ThreadPoolWorkQueue.TransferAllLocalWorkItemsToHighPriorityGlobalQueue(); var mres = new SetOnInvokeMres(); try diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs index 67ccbf4d7cdc06..44e64295db53df 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs @@ -621,8 +621,8 @@ internal void EnsureThreadRequested() // Only request a thread if the stage is NotScheduled. // Otherwise let the current requested thread handle parallelization. if (Interlocked.Exchange( - ref _separated.queueProcessingStage, - QueueProcessingStage.Scheduled) == QueueProcessingStage.NotScheduled) + ref _separated.queueProcessingStage, + QueueProcessingStage.Scheduled) == QueueProcessingStage.NotScheduled) { ThreadPool.RequestWorkerThread(); } @@ -723,14 +723,27 @@ internal static void TransferAllLocalWorkItemsToHighPriorityGlobalQueue() // Pop each work item off the local queue and push it onto the global. This is a // bounded loop as no other thread is allowed to push into this thread's queue. ThreadPoolWorkQueue queue = ThreadPool.s_workQueue; + bool anyWorkItemMoved = false; while (tl.workStealingQueue.LocalPop() is object workItem) { - queue.highPriorityWorkItems.Enqueue(workItem); - } + try + { + queue.highPriorityWorkItems.Enqueue(workItem); + } + catch (Exception ex) + { + // Typically would be an OOM, but since we have lost a work item, fail-fast on any exception + Environment.FailFast("Failed to move a local work item to the high-priority global queue.", ex); + } - Volatile.Write(ref queue._mayHaveHighPriorityWorkItems, true); + anyWorkItemMoved = true; + } - queue.EnsureThreadRequested(); + if (anyWorkItemMoved) + { + Volatile.Write(ref queue._mayHaveHighPriorityWorkItems, true); + queue.EnsureThreadRequested(); + } } internal static bool LocalFindAndPop(object callback) From 0a0865bdd97224280c6530c47dd084b7b9275c56 Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Fri, 21 Feb 2025 13:09:17 -0800 Subject: [PATCH 2/3] Address feedback --- .../src/System/Threading/ThreadPoolWorkQueue.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs index 44e64295db53df..e1deaef2b91188 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs @@ -726,14 +726,18 @@ internal static void TransferAllLocalWorkItemsToHighPriorityGlobalQueue() bool anyWorkItemMoved = false; while (tl.workStealingQueue.LocalPop() is object workItem) { + // If there's an unexpected exception here that happens to get handled, the lost work item, or missing thread + // request, etc., may lead to other issues. A fail-fast or try-finally here could reduce the effect of such + // uncommon issues to various degrees, but it's also uncommon to check for unexpected exceptions. try { queue.highPriorityWorkItems.Enqueue(workItem); } - catch (Exception ex) + catch (OutOfMemoryException) { - // Typically would be an OOM, but since we have lost a work item, fail-fast on any exception - Environment.FailFast("Failed to move a local work item to the high-priority global queue.", ex); + // This is not expected to throw under normal circumstances + tl.workStealingQueue.LocalPush(workItem); + break; } anyWorkItemMoved = true; From 20d308c13d5113df02d6d710ef912d6230ed5a90 Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Mon, 24 Feb 2025 06:31:21 -0800 Subject: [PATCH 3/3] Fix OOM path to ensure a thread request --- .../src/System/Threading/ThreadPoolWorkQueue.cs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs index e1deaef2b91188..d15262a76c27e0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPoolWorkQueue.cs @@ -723,7 +723,8 @@ internal static void TransferAllLocalWorkItemsToHighPriorityGlobalQueue() // Pop each work item off the local queue and push it onto the global. This is a // bounded loop as no other thread is allowed to push into this thread's queue. ThreadPoolWorkQueue queue = ThreadPool.s_workQueue; - bool anyWorkItemMoved = false; + bool addedHighPriorityWorkItem = false; + bool ensureThreadRequest = false; while (tl.workStealingQueue.LocalPop() is object workItem) { // If there's an unexpected exception here that happens to get handled, the lost work item, or missing thread @@ -732,20 +733,28 @@ internal static void TransferAllLocalWorkItemsToHighPriorityGlobalQueue() try { queue.highPriorityWorkItems.Enqueue(workItem); + addedHighPriorityWorkItem = true; } catch (OutOfMemoryException) { // This is not expected to throw under normal circumstances tl.workStealingQueue.LocalPush(workItem); + + // A work item had been removed temporarily and other threads may have missed stealing it, so ensure that + // there will be a thread request + ensureThreadRequest = true; break; } - - anyWorkItemMoved = true; } - if (anyWorkItemMoved) + if (addedHighPriorityWorkItem) { Volatile.Write(ref queue._mayHaveHighPriorityWorkItems, true); + ensureThreadRequest = true; + } + + if (ensureThreadRequest) + { queue.EnsureThreadRequested(); } }