Recognize a VB exception filter's type test

vbc builds an exception filter as a single non-short-circuiting
expression: `isinst`, then the user's `when` conditions, combined with
`and`. DetectCatchWhenConditionBlocks only knew the branch chain csc
emits, so the type test stayed inside the filter and the handler kept the
`object` variable it has in IL:

    catch (object obj) when ((obj is Exception) & (num2 != 0) & (num == 0))
    {
        ProjectData.SetProjectError((Exception)obj);

A catch type has to derive from Exception, so that does not compile.

The conjunction form is matched too now, lifting the test to the catch
type as the block form already does. The other conjuncts stop running for
a non-matching exception once the test moves, so the filter has to be
pure for this to be invisible; PropagateExceptionVariable then drops the
castclass in the handler:

    catch (Exception ex) when ((num2 != 0) & (num == 0))
    {
        ProjectData.SetProjectError(ex);

Closes #3659

ICSharpCode.Decompiler.Tests/ICSharpCode.Decompiler.Tests.csproj

@@ -224,6 +224,8 @@
     <Compile Remove="TestCases\ILPretty\Issue3729.cs" />
     <None Include="TestCases\ILPretty\Issue3729.cs" />
     <None Include="TestCases\ILPretty\Issue3729.il" />
+    <Compile Remove="TestCases\VBPretty\Issue3659.cs" />
+    <None Include="TestCases\VBPretty\Issue3659.cs" />
     <Compile Remove="TestCases\ILPretty\SpanConversionOperatorMismatch.cs" />
     <None Include="TestCases\ILPretty\SpanConversionOperatorMismatch.cs" />
     <Compile Remove="TestCases\ILPretty\FSharpLoops_Debug.cs" />

ICSharpCode.Decompiler.Tests/TestCases/VBPretty/Issue3659.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Runtime.CompilerServices;
+using Microsoft.VisualBasic.CompilerServices;
+
+public class Issue3659
+{
+	public static void Func(ref Issue3659 obj, object value)
+	{
+	}
+
+	public static int ShowMessage(string a, int b, string c, object d, int e)
+	{
+		return 0;
+	}
+
+	internal void VBFunction(object value)
+	{
+#if OPT || LEGACY_VBC
+		int try0000_dispatch = -1;
+#else
+		int try0001_dispatch = -1;
+#endif
+		int num2 = default;
+		int num = default;
+		while (true)
+		{
+			try
+			{
+				/*Note: ILSpy has introduced the following switch to emulate a goto from catch-block to try-block*/;
+#if OPT || LEGACY_VBC
+				switch (try0000_dispatch)
+#else
+				switch (try0001_dispatch)
+#endif
+				{
+					default:
+					{
+						ProjectData.ClearProjectError();
+						num2 = 2;
+						Issue3659 obj = this;
+						Func(ref obj, RuntimeHelpers.GetObjectValue(value));
+#if OPT || LEGACY_VBC
+						goto end_IL_0000;
+#else
+						goto end_IL_0001;
+#endif
+					}
+#if OPT || LEGACY_VBC
+					case 45:
+#else
+					case 50:
+#endif
+						num = -1;
+						switch (num2)
+						{
+							case 2:
+								ShowMessage("VBFunction", 0, "Exception", null, 0);
+#if OPT || LEGACY_VBC
+								goto end_IL_0000;
+#else
+								goto end_IL_0001;
+#endif
+						}
+						break;
+				}
+			}
+			catch (Exception ex) when ((num2 != 0) & (num == 0))
+			{
+				ProjectData.SetProjectError(ex);
+#if OPT || LEGACY_VBC
+				try0000_dispatch = 45;
+#else
+				try0001_dispatch = 50;
+#endif
+				continue;
+			}
+			throw ProjectData.CreateProjectError(-2146828237);
+			continue;
+#if OPT || LEGACY_VBC
+			end_IL_0000:
+#else
+			end_IL_0001:
+#endif
+			break;
+		}
+		if (num != 0)
+		{
+			ProjectData.ClearProjectError();
+		}
+	}
+}

ICSharpCode.Decompiler.Tests/TestCases/VBPretty/Issue3659.vb

@@ -0,0 +1,17 @@
+Imports System
+Public Class Issue3659
+	Public Shared Sub Func(ByRef obj As Issue3659, value As Object)
+	End Sub
+
+	Public Shared Function ShowMessage(a As String, b As Integer, c As String, d As Object, e As Integer) As Integer
+		Return 0
+	End Function
+
+	Friend Sub VBFunction(value As Object)
+		On Error GoTo Handler
+		Func(Me, value)
+		Exit Sub
+Handler:
+		ShowMessage("VBFunction", 0, "Exception", Nothing, 0)
+	End Sub
+End Class

ICSharpCode.Decompiler.Tests/VBPrettyTestRunner.cs

@@ -132,6 +132,12 @@ namespace ICSharpCode.Decompiler.Tests
 			await Run(options: options | CompilerOptions.Library);
 		}
 
+		[Test]
+		public async Task Issue3659([ValueSource(nameof(defaultOptions))] CompilerOptions options)
+		{
+			await Run(options: options | CompilerOptions.Library);
+		}
+
 		[Test]
 		public async Task Issue1906([ValueSource(nameof(defaultOptions))] CompilerOptions options)
 		{

ICSharpCode.Decompiler/IL/Transforms/DetectCatchWhenConditionBlocks.cs

@@ -71,7 +71,75 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 
 					PropagateExceptionVariable(context, catchBlock);
 				}
+				else if (catchBlock.Filter is BlockContainer filterContainer
+					&& MatchTypeTestConjunction(catchBlock.Variable, filterContainer,
+						out var conjunctionType, out var typeTest)
+					&& conjunctionType.GetStackType() == catchBlock.Variable.StackType)
+				{
+					context.Step($"Detected catch-when type test for {catchBlock.Variable.Name}", typeTest);
+					catchBlock.Variable.Type = conjunctionType;
+					var remainder = typeTest.Right;
+					typeTest.ReplaceWith(remainder);
+					context.EndStep(remainder);
+
+					PropagateExceptionVariable(context, catchBlock);
+				}
+			}
+		}
+
+		/// <summary>
+		/// BlockContainer {
+		/// 	Block entryPoint (incoming: 1) {
+		/// 		leave container(bit.and(bit.and(comp(isinst T(ldloc exceptionVar) != ldnull), cond1), cond2))
+		/// 	}
+		/// }
+		/// The Visual Basic compiler emits the whole filter as one non-short-circuiting
+		/// expression rather than the block chain csc emits, so the type test sits in a
+		/// conjunction instead of a branch. On a match, typeTest is the innermost `bit.and`,
+		/// whose right operand is the filter that remains once the test moves to the catch type.
+		/// </summary>
+		bool MatchTypeTestConjunction(ILVariable exceptionVar, BlockContainer container,
+			out IType exceptionType, out BinaryNumericInstruction typeTest)
+		{
+			exceptionType = null;
+			typeTest = null;
+			var entryPoint = container.EntryPoint;
+			if (entryPoint == null || entryPoint.IncomingEdgeCount != 1 || entryPoint.Instructions.Count != 1)
+				return false;
+			if (!entryPoint.Instructions[0].MatchLeave(container, out var condition))
+				return false;
+			// Once the test moves to the catch type, the other conjuncts stop running for a
+			// non-matching exception, which is only invisible if they have no side effects.
+			if (!SemanticHelper.IsPure(condition.Flags))
+				return false;
+			// The test is the leftmost operand of a left-nested chain of `&`.
+			while (condition is BinaryNumericInstruction { Operator: BinaryNumericOperator.BitAnd } and)
+			{
+				if (MatchTypeTest(and.Left, exceptionVar, out exceptionType))
+				{
+					typeTest = and;
+					return true;
 				}
+				condition = and.Left;
+			}
+			return false;
+		}
+
+		/// <summary>
+		/// comp(isinst T(ldloc exceptionVar) != ldnull), however the compiler spelled the null test.
+		/// </summary>
+		static bool MatchTypeTest(ILInstruction condition, ILVariable exceptionVar, out IType exceptionType)
+		{
+			exceptionType = null;
+			if (condition is not Comp comp || !comp.Right.MatchLdNull())
+				return false;
+			// `cgt.un x, null` is how both compilers spell `x != null` for a reference.
+			if (comp.Kind != ComparisonKind.Inequality
+				&& !(comp.Kind == ComparisonKind.GreaterThan && comp.InputType == StackType.O))
+				return false;
+			if (!comp.Left.MatchIsInst(out var argument, out exceptionType))
+				return false;
+			return argument.MatchLdLoc(exceptionVar);
 		}
 
 		/// <summary>