Fix StackOverflowException when subtyping does not terminate due to expansive inheritance.

ICSharpCode.NRefactory.CSharp/Resolver/Conversions.cs

@@ -635,18 +635,33 @@ namespace ICSharpCode.NRefactory.CSharp.Resolver
 		
 		// Determines whether s is a subtype of t.
 		// Helper method used for ImplicitReferenceConversion, BoxingConversion and ImplicitTypeParameterConversion
+		
+		int subtypeCheckNestingDepth;
+		
 		bool IsSubtypeOf(IType s, IType t)
 		{
 			// conversion to dynamic + object are always possible
 			if (t.Equals(SharedTypes.Dynamic) || t.Equals(objectType))
 				return true;
-			
-			// let GetAllBaseTypes do the work for us
-			foreach (IType baseType in s.GetAllBaseTypes(context)) {
-				if (IdentityOrVarianceConversion(baseType, t))
-					return true;
+			try {
+				if (++subtypeCheckNestingDepth > 10) {
+					// Subtyping in C# is undecidable
+					// (see "On Decidability of Nominal Subtyping with Variance" by Andrew J. Kennedy and Benjamin C. Pierce),
+					// so we'll prevent infinite recursions by putting a limit on the nesting depth of variance conversions.
+					
+					// No real C# code should use generics nested more than 10 levels deep, and even if they do, most of
+					// those nestings should not involve variance.
+					return false;
+				}
+				// let GetAllBaseTypes do the work for us
+				foreach (IType baseType in s.GetAllBaseTypes(context)) {
+					if (IdentityOrVarianceConversion(baseType, t))
+						return true;
+				}
+				return false;
+			} finally {
+				subtypeCheckNestingDepth--;
 			}
-			return false;
 		}
 		
 		bool IdentityOrVarianceConversion(IType s, IType t)

ICSharpCode.NRefactory.Tests/CSharp/Resolver/ConversionsTest.cs

@@ -478,5 +478,24 @@ namespace ICSharpCode.NRefactory.CSharp.Resolver
 			Assert.AreEqual(2, BetterConversion(typeof(ushort?), typeof(long?), typeof(int?)));
 			Assert.AreEqual(0, BetterConversion(typeof(sbyte), typeof(int?), typeof(uint?)));
 		}
+		
+		[Test]
+		public void ExpansiveInheritance()
+		{
+			SimpleProjectContent pc = new SimpleProjectContent();
+			DefaultTypeDefinition a = new DefaultTypeDefinition(pc, string.Empty, "A");
+			DefaultTypeDefinition b = new DefaultTypeDefinition(pc, string.Empty, "B");
+			// interface A<in U>
+			a.Kind = TypeKind.Interface;
+			a.TypeParameters.Add(new DefaultTypeParameter(EntityType.TypeDefinition, 0, "U") { Variance = VarianceModifier.Contravariant });
+			// interface B<X> : A<A<B<X>>> { }
+			DefaultTypeParameter x = new DefaultTypeParameter(EntityType.TypeDefinition, 0, "X");
+			b.TypeParameters.Add(x);
+			b.BaseTypes.Add(new ParameterizedType(a, new[] { new ParameterizedType(a, new [] { new ParameterizedType(b, new [] { x }) } ) }));
+			
+			IType type1 = new ParameterizedType(b, new[] { KnownTypeReference.Double.Resolve(ctx) });
+			IType type2 = new ParameterizedType(a, new [] { new ParameterizedType(b, new[] { KnownTypeReference.String.Resolve(ctx) }) });
+			Assert.IsFalse(conversions.ImplicitConversion(type1, type2));
+		}
 	}
 }