[release/11.0] Fix software EH memory leak on Unix

src/coreclr/vm/exceptionhandling.cpp

@@ -976,7 +976,12 @@ VOID DECLSPEC_NORETURN DispatchManagedException(PAL_SEHException& ex, bool isHar
     }
     GCX_COOP();
     OBJECTREF throwable = ExInfo::CreateThrowable(ex.GetExceptionRecord(), FALSE);
-    DispatchManagedException(throwable, ex.GetContextRecord());
+    ExInfo exInfo(GetThread(), ex.GetExceptionRecord(), ex.GetContextRecord(), ExKind::Throw);
+    if (!ex.RecordsOnStack)
+    {
+        exInfo.TakeExceptionPointersOwnership(&ex);
+    }
+    DispatchManagedException(throwable, &exInfo);
 }
 
 #if defined(TARGET_AMD64) || defined(TARGET_X86)
@@ -1752,8 +1757,6 @@ VOID DECLSPEC_NORETURN DispatchManagedException(OBJECTREF throwable, CONTEXT* pE
 
     GCPROTECT_BEGIN(throwable);
 
-   _ASSERTE(IsException(throwable->GetMethodTable()));
-
     Thread *pThread = GetThread();
 
     ULONG_PTR hr = GetHRFromThrowable(throwable);
@@ -1792,6 +1795,25 @@ VOID DECLSPEC_NORETURN DispatchManagedException(OBJECTREF throwable, CONTEXT* pE
     }
 #endif // HOST_WINDOWS
 
+    DispatchManagedException(throwable, &exInfo);
+
+    GCPROTECT_END();
+
+    UNREACHABLE();
+}
+
+VOID DECLSPEC_NORETURN DispatchManagedException(OBJECTREF throwable, ExInfo* pExInfo)
+{
+    STATIC_CONTRACT_THROWS;
+    STATIC_CONTRACT_GC_TRIGGERS;
+    STATIC_CONTRACT_MODE_COOPERATIVE;
+
+    GCPROTECT_BEGIN(throwable);
+
+    _ASSERTE(IsException(throwable->GetMethodTable()));
+
+    Thread *pThread = GetThread();
+
     if (pThread->IsAbortInitiated () && IsExceptionOfType(kThreadAbortException,&throwable))
     {
         pThread->ResetPreparingAbort();
@@ -1810,10 +1832,10 @@ VOID DECLSPEC_NORETURN DispatchManagedException(OBJECTREF throwable, CONTEXT* pE
 
     pThread->IncPreventAbort();
 
-    //Ex.RhThrowEx(throwable, &exInfo)
-    throwEx.InvokeDirect(&throwable, &exInfo);
+    //Ex.RhThrowEx(throwable, pExInfo)
+    throwEx.InvokeDirect(&throwable, pExInfo);
 
-    DispatchExSecondPass(&exInfo);
+    DispatchExSecondPass(pExInfo);
 
     GCPROTECT_END();
 

src/coreclr/vm/exceptionhandling.h

@@ -29,6 +29,7 @@ CallDescrWorkerUnwindFrameChainHandler(IN     PEXCEPTION_RECORD     pExceptionRe
 
 void NormalizeThrownObject(OBJECTREF *ppThrowable);
 
+VOID DECLSPEC_NORETURN DispatchManagedException(OBJECTREF throwable, ExInfo *pExInfo);
 VOID DECLSPEC_NORETURN DispatchManagedException(OBJECTREF throwable, CONTEXT *pExceptionContext, EXCEPTION_RECORD *pExceptionRecord = NULL, ExKind exKind = ExKind::None);
 VOID DECLSPEC_NORETURN DispatchManagedException(OBJECTREF throwable, ExKind exKind = ExKind::None);
 VOID DECLSPEC_NORETURN DispatchManagedException(RuntimeExceptionKind reKind);

src/tests/Regressions/coreclr/GitHub_135234/test135234.cs

@@ -0,0 +1,109 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using Xunit;
+
+public class Runtime_135234
+{
+    private const int WarmupIterations = 10_000;
+    private const int TestIterations = 50_000;
+    private const long MaxMemoryGrowth = 64 * 1024 * 1024;
+
+    private sealed class Instance
+    {
+        public int Value;
+    }
+
+    private static class FailingInitializer
+    {
+        public static int Value;
+
+        static FailingInitializer()
+        {
+            throw new InvalidOperationException("Initialization failed");
+        }
+    }
+
+    [Theory]
+    [SkipOnPlatform(TestPlatforms.Browser | TestPlatforms.Wasi | TestPlatforms.iOS | TestPlatforms.tvOS,
+        "Process working set monitoring is not supported on these platforms.")]
+    [InlineData(false)]
+    [InlineData(true)]
+    public static void TestEntryPoint(bool hardwareException)
+    {
+        using Process process = Process.GetCurrentProcess();
+        RunIterations(WarmupIterations, hardwareException);
+        long before = GetWorkingSet(process);
+
+        RunIterations(TestIterations, hardwareException);
+        long growth = GetWorkingSet(process) - before;
+
+        string exceptionName = hardwareException ? nameof(NullReferenceException) : nameof(TypeInitializationException);
+        Console.WriteLine($"Working set growth after {TestIterations} {exceptionName}s: {growth} bytes");
+        Assert.True(growth < MaxMemoryGrowth, $"Working set grew by {growth} bytes; limit is {MaxMemoryGrowth} bytes.");
+    }
+
+    private static void RunIterations(int count, bool hardwareException)
+    {
+        for (int i = 0; i < count; i++)
+        {
+            if (hardwareException)
+            {
+                CheckHardwareException();
+            }
+            else
+            {
+                CheckException();
+            }
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void CheckException()
+    {
+        try
+        {
+            _ = FailingInitializer.Value;
+        }
+        catch (TypeInitializationException exception)
+        {
+            Assert.IsType<InvalidOperationException>(exception.InnerException);
+            Assert.Equal("Initialization failed", exception.InnerException.Message);
+            return;
+        }
+
+        Assert.Fail("Expected a TypeInitializationException.");
+    }
+
+    private static void CheckHardwareException()
+    {
+        try
+        {
+            _ = ReadField(null);
+        }
+        catch (NullReferenceException)
+        {
+            return;
+        }
+
+        Assert.Fail("Expected a NullReferenceException.");
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static int ReadField(Instance instance)
+    {
+        return instance.Value;
+    }
+
+    private static long GetWorkingSet(Process process)
+    {
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        process.Refresh();
+        return process.WorkingSet64;
+    }
+}

src/tests/Regressions/coreclr/GitHub_135234/test135234.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <RequiresProcessIsolation>true</RequiresProcessIsolation>
+    <GCStressIncompatible>true</GCStressIncompatible>
+    <CLRTestPriority>1</CLRTestPriority>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="test135234.cs" />
+    <CLRTestEnvironmentVariable Include="DOTNET_TieredCompilation" Value="0" />
+  </ItemGroup>
+</Project>