Add support for UTF8 string literals

ICSharpCode.Decompiler.Tests/TestCases/Pretty/InitializerTests.cs

@@ -382,6 +382,9 @@ namespace ICSharpCode.Decompiler.Tests.TestCases.Pretty.InitializerTests
 		public static ReadOnlySpan<byte> StaticData3 => new byte[3] { 1, 2, 3 };
 
 		public static Span<byte> StaticData3Span => new byte[3] { 1, 2, 3 };
+#endif
+#if CS110 && !NET40
+		public static ReadOnlySpan<byte> UTF8Literal => "Hello, world!"u8;
 #endif
 		#endregion
 

ICSharpCode.Decompiler/CSharp/ExpressionBuilder.cs

@@ -617,6 +617,14 @@ namespace ICSharpCode.Decompiler.CSharp
 				.WithRR(new ConstantResolveResult(compilation.FindType(KnownTypeCode.String), inst.Value));
 		}
 
+		protected internal override TranslatedExpression VisitLdStrUtf8(LdStrUtf8 inst, TranslationContext context)
+		{
+			var type = new ParameterizedType(compilation.FindType(KnownTypeCode.ReadOnlySpanOfT), new[] { compilation.FindType(KnownTypeCode.Byte) });
+			return new PrimitiveExpression(inst.Value, LiteralFormat.Utf8Literal)
+				.WithILInstruction(inst)
+				.WithRR(new ConstantResolveResult(type, inst.Value));
+		}
+
 		protected internal override TranslatedExpression VisitLdNull(LdNull inst, TranslationContext context)
 		{
 			return GetDefaultValueExpression(SpecialType.NullType).WithILInstruction(inst);

ICSharpCode.Decompiler/CSharp/OutputVisitor/InsertRequiredSpacesDecorator.cs

@@ -159,7 +159,10 @@ namespace ICSharpCode.Decompiler.CSharp.OutputVisitor
 				return;
 			if (value is string)
 			{
-				lastWritten = LastWritten.Other;
+				if (format == LiteralFormat.VerbatimStringLiteral)
+					lastWritten = LastWritten.KeywordOrIdentifier;
+				else
+					lastWritten = LastWritten.Other;
 			}
 			else if (value is char)
 			{

ICSharpCode.Decompiler/CSharp/OutputVisitor/TextWriterTokenWriter.cs

@@ -269,6 +269,12 @@ namespace ICSharpCode.Decompiler.CSharp.OutputVisitor
 				textWriter.Write('"');
 				textWriter.Write(tmp);
 				textWriter.Write('"');
+				if (format == LiteralFormat.Utf8Literal)
+				{
+					textWriter.Write("u8");
+					column += 2;
+					Length += 2;
+				}
 			}
 			else if (value is char)
 			{

ICSharpCode.Decompiler/CSharp/Syntax/Expressions/PrimitiveExpression.cs

@@ -42,6 +42,7 @@ namespace ICSharpCode.Decompiler.CSharp.Syntax
 		StringLiteral,
 		VerbatimStringLiteral,
 		CharLiteral,
+		Utf8Literal,
 	}
 
 	/// <summary>

ICSharpCode.Decompiler/DecompilerSettings.cs

@@ -153,12 +153,13 @@ namespace ICSharpCode.Decompiler
 				lifetimeAnnotations = false;
 				requiredMembers = false;
 				numericIntPtr = false;
+				utf8StringLiterals = false;
 			}
 		}
 
 		public CSharp.LanguageVersion GetMinimumRequiredVersion()
 		{
-			if (parameterNullCheck || lifetimeAnnotations || requiredMembers || numericIntPtr)
+			if (parameterNullCheck || lifetimeAnnotations || requiredMembers || numericIntPtr || utf8StringLiterals)
 				return CSharp.LanguageVersion.CSharp11_0;
 			if (fileScopedNamespaces || recordStructs)
 				return CSharp.LanguageVersion.CSharp10_0;
@@ -434,9 +435,10 @@ namespace ICSharpCode.Decompiler
 		/// <summary>
 		/// Use C# 11 preview parameter null-checking (<code>string param!!</code>).
 		/// </summary>
-		[Category("C# 11.0 / VS 2022.1")]
+		[Category("C# 11.0 / VS 2022.4")]
 		[Description("DecompilerSettings.ParameterNullCheck")]
 		[Browsable(false)]
+		[Obsolete("This feature did not make it into C# 11, and may be removed in a future version of the decompiler.")]
 		public bool ParameterNullCheck {
 			get { return parameterNullCheck; }
 			set {
@@ -1173,6 +1175,24 @@ namespace ICSharpCode.Decompiler
 			}
 		}
 
+		bool utf8StringLiterals = true;
+
+		/// <summary>
+		/// Gets/Sets whether to use C# 11.0 UTF-8 string literals
+		/// </summary>
+		[Category("C# 11.0 / VS 2022.4")]
+		[Description("DecompilerSettings.Utf8StringLiterals")]
+		public bool Utf8StringLiterals {
+			get { return utf8StringLiterals; }
+			set {
+				if (utf8StringLiterals != value)
+				{
+					utf8StringLiterals = value;
+					OnPropertyChanged();
+				}
+			}
+		}
+
 		bool showXmlDocumentation = true;
 
 		/// <summary>

ICSharpCode.Decompiler/IL/Instructions.cs

@@ -19,6 +19,7 @@
 #nullable enable
 
 using System;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 
@@ -121,6 +122,8 @@ namespace ICSharpCode.Decompiler.IL
 		NullableRewrap,
 		/// <summary>Loads a constant string.</summary>
 		LdStr,
+		/// <summary>Loads a constant byte string (as ReadOnlySpan&lt;byte&gt;).</summary>
+		LdStrUtf8,
 		/// <summary>Loads a constant 32-bit integer.</summary>
 		LdcI4,
 		/// <summary>Loads a constant 64-bit integer.</summary>
@@ -2838,6 +2841,43 @@ namespace ICSharpCode.Decompiler.IL
 	}
 }
 namespace ICSharpCode.Decompiler.IL
+{
+	/// <summary>Loads a constant byte string (as ReadOnlySpan&lt;byte&gt;).</summary>
+	public sealed partial class LdStrUtf8 : SimpleInstruction
+	{
+		public LdStrUtf8(string value) : base(OpCode.LdStrUtf8)
+		{
+			this.Value = value;
+		}
+		public readonly string Value;
+		public override StackType ResultType { get { return StackType.O; } }
+		public override void WriteTo(ITextOutput output, ILAstWritingOptions options)
+		{
+			WriteILRange(output, options);
+			output.Write(OpCode);
+			output.Write(' ');
+			Disassembler.DisassemblerHelpers.WriteOperand(output, Value);
+		}
+		public override void AcceptVisitor(ILVisitor visitor)
+		{
+			visitor.VisitLdStrUtf8(this);
+		}
+		public override T AcceptVisitor<T>(ILVisitor<T> visitor)
+		{
+			return visitor.VisitLdStrUtf8(this);
+		}
+		public override T AcceptVisitor<C, T>(ILVisitor<C, T> visitor, C context)
+		{
+			return visitor.VisitLdStrUtf8(this, context);
+		}
+		protected internal override bool PerformMatch(ILInstruction? other, ref Patterns.Match match)
+		{
+			var o = other as LdStrUtf8;
+			return o != null && this.Value == o.Value;
+		}
+	}
+}
+namespace ICSharpCode.Decompiler.IL
 {
 	/// <summary>Loads a constant 32-bit integer.</summary>
 	public sealed partial class LdcI4 : SimpleInstruction
@@ -6947,6 +6987,10 @@ namespace ICSharpCode.Decompiler.IL
 		{
 			Default(inst);
 		}
+		protected internal virtual void VisitLdStrUtf8(LdStrUtf8 inst)
+		{
+			Default(inst);
+		}
 		protected internal virtual void VisitLdcI4(LdcI4 inst)
 		{
 			Default(inst);
@@ -7345,6 +7389,10 @@ namespace ICSharpCode.Decompiler.IL
 		{
 			return Default(inst);
 		}
+		protected internal virtual T VisitLdStrUtf8(LdStrUtf8 inst)
+		{
+			return Default(inst);
+		}
 		protected internal virtual T VisitLdcI4(LdcI4 inst)
 		{
 			return Default(inst);
@@ -7743,6 +7791,10 @@ namespace ICSharpCode.Decompiler.IL
 		{
 			return Default(inst, context);
 		}
+		protected internal virtual T VisitLdStrUtf8(LdStrUtf8 inst, C context)
+		{
+			return Default(inst, context);
+		}
 		protected internal virtual T VisitLdcI4(LdcI4 inst, C context)
 		{
 			return Default(inst, context);
@@ -8013,6 +8065,7 @@ namespace ICSharpCode.Decompiler.IL
 			"nullable.unwrap",
 			"nullable.rewrap",
 			"ldstr",
+			"ldstr.utf8",
 			"ldc.i4",
 			"ldc.i8",
 			"ldc.f4",
@@ -8285,6 +8338,17 @@ namespace ICSharpCode.Decompiler.IL
 			value = default(string);
 			return false;
 		}
+		public bool MatchLdStrUtf8([NotNullWhen(true)] out string? value)
+		{
+			var inst = this as LdStrUtf8;
+			if (inst != null)
+			{
+				value = inst.Value;
+				return true;
+			}
+			value = default(string);
+			return false;
+		}
 		public bool MatchLdcI4(out int value)
 		{
 			var inst = this as LdcI4;
@@ -8751,3 +8815,4 @@ namespace ICSharpCode.Decompiler.IL
 		}
 	}
 }
+

ICSharpCode.Decompiler/IL/Instructions.tt

@@ -204,6 +204,8 @@
 
 		new OpCode("ldstr", "Loads a constant string.",
 			CustomClassName("LdStr"), LoadConstant("string"), ResultType("O")),
+		new OpCode("ldstr.utf8", "Loads a constant byte string (as ReadOnlySpan<byte>).",
+			CustomClassName("LdStrUtf8"), LoadConstant("string"), ResultType("O")),
 		new OpCode("ldc.i4", "Loads a constant 32-bit integer.",
 			LoadConstant("int"), ResultType("I4")),
 		new OpCode("ldc.i8", "Loads a constant 64-bit integer.",

ICSharpCode.Decompiler/IL/Transforms/ExpressionTransforms.cs

@@ -313,10 +313,10 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 					ILInlining.InlineIfPossible(block, stmt.ChildIndex, context);
 				return;
 			}
-			if (TransformArrayInitializers.TransformSpanTArrayInitialization(inst, context, out block))
+			if (TransformArrayInitializers.TransformSpanTArrayInitialization(inst, context, out var replacement))
 			{
 				context.Step("TransformSpanTArrayInitialization: single-dim", inst);
-				inst.ReplaceWith(block);
+				inst.ReplaceWith(replacement);
 				return;
 			}
 			if (TransformDelegateCtorLdVirtFtnToLdVirtDelegate(inst, out LdVirtDelegate ldVirtDelegate))

ICSharpCode.Decompiler/IL/Transforms/ParameterNullCheckTransform.cs

@@ -31,8 +31,10 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 	{
 		void IILTransform.Run(ILFunction function, ILTransformContext context)
 		{
+#pragma warning disable 618 // ParameterNullCheck is obsolete
 			if (!context.Settings.ParameterNullCheck)
 				return;
+#pragma warning restore 618
 			// we only need to look at the entry-point as parameter null-checks
 			// do not produce any IL control-flow instructions
 			Block entryPoint = ((BlockContainer)function.Body).EntryPoint;

ICSharpCode.Decompiler/IL/Transforms/TransformArrayInitializers.cs

@@ -20,6 +20,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection.Metadata;
+using System.Text;
 
 using ICSharpCode.Decompiler.TypeSystem;
 using ICSharpCode.Decompiler.Util;
@@ -113,9 +114,9 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 			return false;
 		}
 
-		internal static bool TransformSpanTArrayInitialization(NewObj inst, StatementTransformContext context, out Block block)
+		internal static bool TransformSpanTArrayInitialization(NewObj inst, StatementTransformContext context, out ILInstruction replacement)
 		{
-			block = null;
+			replacement = null;
 			if (!context.Settings.ArrayInitializers)
 				return false;
 			if (MatchSpanTCtorWithPointerAndSize(inst, context, out var elementType, out var field, out var size))
@@ -124,10 +125,17 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 				{
 					var valuesList = new List<ILInstruction>();
 					var initialValue = field.GetInitialValue(context.PEFile.Reader, context.TypeSystem);
+					if (context.Settings.Utf8StringLiterals &&
+						elementType.IsKnownType(KnownTypeCode.Byte) &&
+						DecodeUTF8String(initialValue, size, out string text))
+					{
+						replacement = new LdStrUtf8(text);
+						return true;
+					}
 					if (DecodeArrayInitializer(elementType, initialValue, new[] { size }, valuesList))
 					{
 						var tempStore = context.Function.RegisterVariable(VariableKind.InitializerTarget, new ArrayType(context.TypeSystem, elementType));
-						block = BlockFromInitializer(tempStore, elementType, new[] { size }, valuesList.ToArray());
+						replacement = BlockFromInitializer(tempStore, elementType, new[] { size }, valuesList.ToArray());
 						return true;
 					}
 				}
@@ -135,13 +143,36 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 			return false;
 		}
 
+		private static unsafe bool DecodeUTF8String(BlobReader blob, int size, out string text)
+		{
+			if (size > blob.RemainingBytes)
+			{
+				text = null;
+				return false;
+			}
+			for (int i = 0; i < size; i++)
+			{
+				byte val = blob.CurrentPointer[i];
+				// If the string has control characters, it's probably binary data and not a string.
+				if (val < 0x20 && val is not ((byte)'\r' or (byte)'\n' or (byte)'\t'))
+				{
+					text = null;
+					return false;
+				}
+			}
+			text = Encoding.UTF8.GetString(blob.CurrentPointer, size);
+			// Only use UTF8 string literal if we can perfectly roundtrip the data
+			byte[] bytes = Encoding.UTF8.GetBytes(text);
+			return MemoryExtensions.SequenceEqual(bytes, new ReadOnlySpan<byte>(blob.CurrentPointer, size));
+		}
+
 		static bool MatchSpanTCtorWithPointerAndSize(NewObj newObj, StatementTransformContext context, out IType elementType, out FieldDefinition field, out int size)
 		{
 			field = default;
 			size = default;
 			elementType = null;
 			IType type = newObj.Method.DeclaringType;
-			if (!type.IsKnownType(KnownTypeCode.SpanOfT) && !type.IsKnownType(KnownTypeCode.ReadOnlySpanOfT))
+			if (!type.IsKnownType(KnownTypeCode.ReadOnlySpanOfT))
 				return false;
 			if (newObj.Arguments.Count != 2 || type.TypeArguments.Count != 1)
 				return false;

ICSharpCode.Decompiler/IL/Transforms/UsingTransform.cs

@@ -390,7 +390,7 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 			{
 				if (left.MatchStLoc(out var inlineAssignVar, out var inlineAssignVal))
 				{
-					if (!inlineAssignVal.MatchIsInst(out var arg, out var type) && type.IsKnownType(disposeType))
+					if (!inlineAssignVal.MatchIsInst(out var arg, out var type) || !type.IsKnownType(disposeType))
 						return false;
 					if (!inlineAssignVar.IsSingleDefinition || inlineAssignVar.LoadCount != 1)
 						return false;

ILSpy/Properties/Resources.Designer.cs

@@ -1,6 +1,7 @@
 //------------------------------------------------------------------------------
 // <auto-generated>
 //     This code was generated by a tool.
+//     Runtime Version:4.0.30319.42000
 //
 //     Changes to this file may cause incorrect behavior and will be lost if
 //     the code is regenerated.
@@ -1477,6 +1478,15 @@ namespace ICSharpCode.ILSpy.Properties {
             }
         }
         
+        /// <summary>
+        ///   Looks up a localized string similar to UTF-8 string literals.
+        /// </summary>
+        public static string DecompilerSettings_Utf8StringLiterals {
+            get {
+                return ResourceManager.GetString("DecompilerSettings.Utf8StringLiterals", resourceCulture);
+            }
+        }
+        
         /// <summary>
         ///   Looks up a localized string similar to VB-specific options.
         /// </summary>

ILSpy/Properties/Resources.resx

@@ -516,6 +516,9 @@ Are you sure you want to continue?</value>
   <data name="DecompilerSettings.UseVariableNamesFromDebugSymbolsIfAvailable" xml:space="preserve">
     <value>Use variable names from debug symbols, if available</value>
   </data>
+  <data name="DecompilerSettings.Utf8StringLiterals" xml:space="preserve">
+    <value>UTF-8 string literals</value>
+  </data>
   <data name="DecompilerSettings.VBSpecificOptions" xml:space="preserve">
     <value>VB-specific options</value>
   </data>
@@ -544,7 +547,7 @@ Are you sure you want to continue?</value>
     <value>Font:</value>
   </data>
   <data name="DisplaySettingsPanel_Theme" xml:space="preserve">
-	  <value>Theme:</value>
+    <value>Theme:</value>
   </data>
   <data name="Download" xml:space="preserve">
     <value>Download</value>
@@ -911,7 +914,7 @@ Do you want to continue?</value>
     <value>Tab size:</value>
   </data>
   <data name="Theme" xml:space="preserve">
-	  <value>Theme</value>
+    <value>Theme</value>
   </data>
   <data name="ToggleFolding" xml:space="preserve">
     <value>Toggle All Folding</value>