Merge pull request #3620 from ds5678/unmanaged-function-pointer-improvements

Improve decompilation of unmanaged function pointers

ICSharpCode.Decompiler.Tests/TestCases/Pretty/FunctionPointers.cs

@@ -1,6 +1,6 @@
-#if !(CS110 && NET70)
 using System;
-#endif
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Text;
 
 namespace ICSharpCode.Decompiler.Tests.TestCases.Pretty
@@ -46,15 +46,168 @@ namespace ICSharpCode.Decompiler.Tests.TestCases.Pretty
 			return (delegate*<object, string>)(&VarianceTest);
 		}
 
-		public unsafe delegate*<void> AddressOfLocalFunction()
+		public unsafe delegate*<void> AddressOfLocalFunction_Managed()
+		{
+			return &LocalFunction;
+
+			static void LocalFunction()
+			{
+			}
+		}
+
+		public unsafe delegate* unmanaged<void> AddressOfLocalFunction_Unmanaged()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly]
+			static void LocalFunction()
+			{
+			}
+		}
+
+		public unsafe delegate* unmanaged[Cdecl]<void> AddressOfLocalFunction_CDecl()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] { typeof(CallConvCdecl) })]
+			static void LocalFunction()
+			{
+			}
+		}
+
+		public unsafe delegate* unmanaged[Fastcall]<void> AddressOfLocalFunction_Fastcall()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] { typeof(CallConvFastcall) })]
+			static void LocalFunction()
+			{
+			}
+		}
+
+#if NET60
+		public unsafe delegate* unmanaged[MemberFunction]<void> AddressOfLocalFunction_MemberFunction()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] { typeof(CallConvMemberFunction) })]
+			static void LocalFunction()
+			{
+			}
+		}
+#endif
+
+		public unsafe delegate* unmanaged[Stdcall]<void> AddressOfLocalFunction_Stdcall()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] { typeof(CallConvStdcall) })]
+			static void LocalFunction()
+			{
+			}
+		}
+
+#if NET60
+		public unsafe delegate* unmanaged[SuppressGCTransition]<void> AddressOfLocalFunction_SuppressGCTransition()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] { typeof(CallConvSuppressGCTransition) })]
+			static void LocalFunction()
+			{
+			}
+		}
+#endif
+
+		public unsafe delegate* unmanaged[Thiscall]<void> AddressOfLocalFunction_Thiscall()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] { typeof(CallConvThiscall) })]
+			static void LocalFunction()
+			{
+			}
+		}
+
+		public unsafe delegate* unmanaged[Cdecl, Fastcall]<void> AddressOfLocalFunction_CDeclAndFastcall()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] {
+				typeof(CallConvCdecl),
+				typeof(CallConvFastcall)
+			})]
+			static void LocalFunction()
+			{
+			}
+		}
+
+		public unsafe delegate* unmanaged[Fastcall, Cdecl]<void> AddressOfLocalFunction_FastcallAndCDecl()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] {
+				typeof(CallConvFastcall),
+				typeof(CallConvCdecl)
+			})]
+			static void LocalFunction()
+			{
+			}
+		}
+
+#if NET60
+		public unsafe delegate* unmanaged[Cdecl, SuppressGCTransition]<void> AddressOfLocalFunction_CDeclAndSuppressGCTransition()
 		{
 			return &LocalFunction;
 
+			[UnmanagedCallersOnly(CallConvs = new Type[] {
+				typeof(CallConvCdecl),
+				typeof(CallConvSuppressGCTransition)
+			})]
 			static void LocalFunction()
 			{
+			}
+		}
+
+		public unsafe delegate* unmanaged[Fastcall, SuppressGCTransition]<void> AddressOfLocalFunction_FastcallAndSuppressGCTransition()
+		{
+			return &LocalFunction;
 
+			[UnmanagedCallersOnly(CallConvs = new Type[] {
+				typeof(CallConvFastcall),
+				typeof(CallConvSuppressGCTransition)
+			})]
+			static void LocalFunction()
+			{
 			}
 		}
+
+		public unsafe delegate* unmanaged[Stdcall, SuppressGCTransition]<void> AddressOfLocalFunction_StdcallAndSuppressGCTransition()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] {
+				typeof(CallConvStdcall),
+				typeof(CallConvSuppressGCTransition)
+			})]
+			static void LocalFunction()
+			{
+			}
+		}
+
+		public unsafe delegate* unmanaged[Thiscall, SuppressGCTransition]<void> AddressOfLocalFunction_ThiscallAndSuppressGCTransition()
+		{
+			return &LocalFunction;
+
+			[UnmanagedCallersOnly(CallConvs = new Type[] {
+				typeof(CallConvThiscall),
+				typeof(CallConvSuppressGCTransition)
+			})]
+			static void LocalFunction()
+			{
+			}
+		}
+#endif
 	}
 
 	internal class FunctionPointersWithCallingConvention

ICSharpCode.Decompiler/CSharp/ExpressionBuilder.cs

@@ -4598,10 +4598,50 @@ namespace ICSharpCode.Decompiler.CSharp
 					.WithILInstruction(inst);
 			}
 			// C# 9 function pointer
+			SignatureCallingConvention callingConvention = SignatureCallingConvention.Default;
+			ImmutableArray<IType> customCallingConventions;
+			var unmanagedCallersOnlyAttribute = inst.Method.GetAttributes().FirstOrDefault(m => m.AttributeType.IsKnownType(KnownAttribute.UnmanagedCallersOnly));
+			if (unmanagedCallersOnlyAttribute != null)
+			{
+				callingConvention = SignatureCallingConvention.Unmanaged;
+
+				var callingConventionsArgument = unmanagedCallersOnlyAttribute.NamedArguments.FirstOrDefault(a => a.Name == "CallConvs");
+				if (callingConventionsArgument.Value is ImmutableArray<CustomAttributeTypedArgument<IType>> array)
+				{
+					var builder = ImmutableArray.CreateBuilder<IType>(array.Length);
+					foreach (var type in array.Select(a => a.Value).OfType<IType>())
+					{
+						SignatureCallingConvention? foundCallingConvention = type.Namespace is not "System.Runtime.CompilerServices" || callingConvention != SignatureCallingConvention.Unmanaged ? null : type.Name switch {
+							"CallConvCdecl" => SignatureCallingConvention.CDecl,
+							"CallConvFastcall" => SignatureCallingConvention.FastCall,
+							"CallConvStdcall" => SignatureCallingConvention.StdCall,
+							"CallConvThiscall" => SignatureCallingConvention.ThisCall,
+							_ => null,
+						};
+						if (foundCallingConvention is not null)
+						{
+							callingConvention = foundCallingConvention.Value;
+						}
+						else
+						{
+							builder.Add(type);
+						}
+					}
+					customCallingConventions = builder.ToImmutable();
+				}
+				else
+				{
+					customCallingConventions = [];
+				}
+			}
+			else
+			{
+				callingConvention = SignatureCallingConvention.Default;
+				customCallingConventions = [];
+			}
 			var ftp = new FunctionPointerType(
 				typeSystem.MainModule,
-				// TODO: calling convention
+				callingConvention, customCallingConventions,
-				SignatureCallingConvention.Default, ImmutableArray.Create<IType>(),
 				inst.Method.ReturnType, inst.Method.ReturnTypeIsRefReadOnly,
 				inst.Method.Parameters.SelectImmutableArray(p => p.Type),
 				inst.Method.Parameters.SelectImmutableArray(p => p.ReferenceKind)

ICSharpCode.Decompiler/CSharp/Syntax/TypeSystemAstBuilder.cs

@@ -368,7 +368,7 @@ namespace ICSharpCode.Decompiler.CSharp.Syntax
 				foreach (var customCallConv in fpt.CustomCallingConventions)
 				{
 					AstType callConvSyntax;
-					if (customCallConv.Name.StartsWith("CallConv", StringComparison.Ordinal) && customCallConv.Name.Length > 8)
+					if (customCallConv.Namespace == "System.Runtime.CompilerServices" && customCallConv.Name.StartsWith("CallConv", StringComparison.Ordinal) && customCallConv.Name.Length > 8)
 					{
 						callConvSyntax = new PrimitiveType(customCallConv.Name.Substring(8));
 						if (AddResolveResultAnnotations)

ICSharpCode.Decompiler/TypeSystem/FunctionPointerType.cs

@@ -33,6 +33,7 @@ namespace ICSharpCode.Decompiler.TypeSystem
 		{
 			IType returnType = signature.ReturnType;
 			bool returnIsRefReadOnly = false;
+			SignatureCallingConvention callingConvention = signature.Header.CallingConvention;
 			var customCallConvs = ImmutableArray.CreateBuilder<IType>();
 			while (returnType is ModifiedType modReturn)
 			{
@@ -45,7 +46,31 @@ namespace ICSharpCode.Decompiler.TypeSystem
 					&& modReturn.Modifier.Namespace == "System.Runtime.CompilerServices")
 				{
 					returnType = modReturn.ElementType;
-					customCallConvs.Add(modReturn.Modifier);
+					if (callingConvention == SignatureCallingConvention.Unmanaged)
+					{
+						switch (modReturn.Modifier.Name)
+						{
+							case "CallConvCdecl":
+								callingConvention = SignatureCallingConvention.CDecl;
+								break;
+							case "CallConvFastcall":
+								callingConvention = SignatureCallingConvention.FastCall;
+								break;
+							case "CallConvStdcall":
+								callingConvention = SignatureCallingConvention.StdCall;
+								break;
+							case "CallConvThiscall":
+								callingConvention = SignatureCallingConvention.ThisCall;
+								break;
+							default:
+								customCallConvs.Add(modReturn.Modifier);
+								break;
+						}
+					}
+					else
+					{
+						customCallConvs.Add(modReturn.Modifier);
+					}
 				}
 				else
 				{
@@ -89,7 +114,7 @@ namespace ICSharpCode.Decompiler.TypeSystem
 				parameterReferenceKinds.Add(kind);
 			}
 			return new FunctionPointerType(
-				module, signature.Header.CallingConvention, customCallConvs.ToImmutable(),
+				module, callingConvention, customCallConvs.ToImmutable(),
 				returnType, returnIsRefReadOnly,
 				parameterTypes.MoveToImmutable(), parameterReferenceKinds.MoveToImmutable());
 		}

ICSharpCode.Decompiler/TypeSystem/Implementation/KnownAttributes.cs

@@ -110,6 +110,7 @@ namespace ICSharpCode.Decompiler.TypeSystem
 		// C# 9 attributes:
 		NativeInteger,
 		PreserveBaseOverrides,
+		UnmanagedCallersOnly,
 
 		// C# 11 attributes:
 		Required,
@@ -192,6 +193,7 @@ namespace ICSharpCode.Decompiler.TypeSystem
 			// C# 9 attributes:
 			new TopLevelTypeName("System.Runtime.CompilerServices", "NativeIntegerAttribute"),
 			new TopLevelTypeName("System.Runtime.CompilerServices", "PreserveBaseOverridesAttribute"),
+			new TopLevelTypeName("System.Runtime.InteropServices", "UnmanagedCallersOnlyAttribute"),
 			// C# 11 attributes:
 			new TopLevelTypeName("System.Runtime.CompilerServices", "RequiredMemberAttribute"),
 			// C# 12 attributes: