Make auto-compiling of C# work with .NET Core

CodeDom doesn't work in .NET Core (https://github.com/dotnet/runtime/issues/18768) and Roslyn turns out to be too low level (just a compiler and not a building framework) so the best approach is to invoke msbuild as a process.

Signed-off-by: Dimitar Dobrev <dpldobrev@protonmail.com>

Directory.Packages.props

@@ -1,6 +1,5 @@
 <Project>
   <ItemGroup>
-    <PackageVersion Include="System.CodeDom" Version="4.7.0" />
     <PackageVersion Include="Microsoft.Win32.Registry" Version="4.7.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Setup.Configuration.Interop" Version="2.3.2262-g94fae01e" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />

build/Tests.lua

@@ -84,7 +84,9 @@ function SetupTestProjectsCSharp(name, depends, extraFiles, suffix)
       str = "Std"
     end
   
-  SetupExternalManagedTestProject(name .. ".CSharp")
+  if name ~= "NamespacesDerived" then
+    SetupExternalManagedTestProject(name .. ".CSharp")
+  end
   SetupExternalManagedTestProject(name .. ".Tests.CSharp")
 end
 

src/Generator/CppSharp.Generator.csproj

@@ -12,8 +12,4 @@
     <ProjectReference Include="..\Parser\CppSharp.Parser.csproj" />
     <ProjectReference Include="$(NativeProjectsDir)Std-symbols.vcxproj" ReferenceOutputAssembly="false" Condition="$(IsWindows)" />
   </ItemGroup>
-  
-  <ItemGroup>
-    <PackageReference Include="System.CodeDom" PrivateAssets="All" />
-  </ItemGroup>
 </Project>

src/Generator/Driver.cs

@@ -1,26 +1,23 @@
 using System;
-using System.CodeDom.Compiler;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text;
 using CppSharp.AST;
 using CppSharp.Generators;
+using CppSharp.Generators.C;
 using CppSharp.Generators.CLI;
+using CppSharp.Generators.Cpp;
 using CppSharp.Generators.CSharp;
 using CppSharp.Parser;
 using CppSharp.Passes;
 using CppSharp.Utils;
-using Microsoft.CSharp;
 using CppSharp.Types;
-using CppSharp.Generators.Cpp;
-using CppSharp.Generators.C;
 
 namespace CppSharp
 {
     public class Driver : IDisposable
     {
-        public DriverOptions Options { get; private set; }
+        public DriverOptions Options { get; }
         public ParserOptions ParserOptions { get; set; }
         public BindingContext Context { get; private set; }
         public Generator Generator { get; private set; }
@@ -351,68 +348,27 @@ namespace CppSharp
                 File.WriteAllText(file, generatedCode);
         }
 
-        private static readonly Dictionary<Module, string> libraryMappings = new Dictionary<Module, string>();
+        public bool CompileCode(Module module)
-
-        public void CompileCode(Module module)
         {
-            var assemblyFile = Path.Combine(Options.OutputDir, module.LibraryName + ".dll");
+            var msBuildGenerator = new MSBuildGenerator(Context, module, libraryMappings);
-
+            msBuildGenerator.Process();
-            var docFile = Path.ChangeExtension(assemblyFile, ".xml");
+            string csproj = Path.Combine(Options.OutputDir,
-
+                $"{module.LibraryName}.{msBuildGenerator.FileExtension}");
-            var compilerOptions = new StringBuilder();
+            File.WriteAllText(csproj, msBuildGenerator.Generate());
-            compilerOptions.Append($" /doc:\"{docFile}\"");
+
-            compilerOptions.Append(" /debug:pdbonly");
+            string output = ProcessHelper.Run("dotnet", $"build {csproj}",
-            compilerOptions.Append(" /unsafe");
+                out int error, out string errorMessage);
-
+            if (error == 0)
-            var compilerParameters = new CompilerParameters
-            {
-                GenerateExecutable = false,
-                TreatWarningsAsErrors = false,
-                OutputAssembly = assemblyFile,
-                GenerateInMemory = false,
-                CompilerOptions = compilerOptions.ToString()
-            };
-
-            if (module != Options.SystemModule)
-                compilerParameters.ReferencedAssemblies.Add(
-                    Path.Combine(Options.OutputDir, $"{Options.SystemModule.LibraryName}.dll"));
-            // add a reference to System.Core
-            compilerParameters.ReferencedAssemblies.Add(typeof(Enumerable).Assembly.Location);
-
-            var location = System.Reflection.Assembly.GetExecutingAssembly().Location;
-            var outputDir = Path.GetDirectoryName(location);
-            var locationRuntime = Path.Combine(outputDir, "CppSharp.Runtime.dll");
-            compilerParameters.ReferencedAssemblies.Add(locationRuntime);
-
-            compilerParameters.ReferencedAssemblies.AddRange(
-                (from dependency in module.Dependencies
-                 where libraryMappings.ContainsKey(dependency)
-                 select libraryMappings[dependency]).ToArray());
-
-            compilerParameters.ReferencedAssemblies.AddRange(module.ReferencedAssemblies.ToArray());
-
-            Diagnostics.Message($"Compiling {module.LibraryName}...");
-            CompilerResults compilerResults;
-            using (var codeProvider = new CSharpCodeProvider(
-                new Dictionary<string, string> {
-                    { "CompilerDirectoryPath", ManagedToolchain.FindCSharpCompilerDir() } }))
             {
-                compilerResults = codeProvider.CompileAssemblyFromFile(
+                Diagnostics.Message($@"Compilation succeeded: {
-                    compilerParameters, module.CodeFiles.ToArray());
+                    libraryMappings[module] = Path.Combine(
+                        Options.OutputDir, $"{module.LibraryName}.dll")}.");
+                return true;
             }
 
-            var errors = compilerResults.Errors.Cast<CompilerError>().Where(e => !e.IsWarning &&
+            Diagnostics.Error(output);
-                // HACK: auto-compiling on OS X produces "errors" which do not affect compilation so we ignore them
+            Diagnostics.Error(errorMessage);
-                (!Platform.IsMacOS || !e.ErrorText.EndsWith("(Location of the symbol related to previous warning)", StringComparison.Ordinal))).ToList();
+            return false;
-            foreach (var error in errors)
-                Diagnostics.Error(error.ToString());
-
-            HasCompilationErrors = errors.Count > 0;
-            if (!HasCompilationErrors)
-            {
-                libraryMappings[module] = Path.Combine(outputDir, assemblyFile);
-                Diagnostics.Message("Compilation succeeded.");
-            }
         }
 
         public void AddTranslationUnitPass(TranslationUnitPass pass)
@@ -433,6 +389,7 @@ namespace CppSharp
         }
 
         private bool hasParsingErrors;
+        private static readonly Dictionary<Module, string> libraryMappings = new Dictionary<Module, string>();
     }
 
     public static class ConsoleDriver
@@ -498,12 +455,7 @@ namespace CppSharp
 
                     driver.SaveCode(outputs);
                     if (driver.Options.IsCSharpGenerator && driver.Options.CompileCode)
-                        foreach (var module in driver.Options.Modules)
+                        driver.Options.Modules.Any(m => !driver.CompileCode(m));
-                        {
-                            driver.CompileCode(module);
-                            if (driver.HasCompilationErrors)
-                                break;
-                        }
                 }
             }
         }

src/Generator/Generators/MSBuildGenerator.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using CppSharp.AST;
+
+namespace CppSharp.Generators
+{
+    public class MSBuildGenerator : CodeGenerator
+    {
+        public MSBuildGenerator(BindingContext context, Module module,
+            Dictionary<Module, string> libraryMappings)
+            : base(context)
+        {
+            this.module = module;
+            this.libraryMappings = libraryMappings;
+        }
+
+        public override string FileExtension => "csproj";
+
+        public override void Process()
+        {
+            var location = System.Reflection.Assembly.GetExecutingAssembly().Location;
+            Write($@"
+<Project Sdk=""Microsoft.NET.Sdk"">
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <PlatformTarget>{(Context.TargetInfo.PointerWidth == 64 ? "x64" : "x86")}</PlatformTarget>
+    <OutputPath>{Options.OutputDir}</OutputPath>
+    <DocumentationFile>{module.LibraryName}.xml</DocumentationFile>
+    <Configuration>Release</Configuration>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <EnableDefaultNoneItems>false</EnableDefaultNoneItems>
+    <EnableDefaultItems>false</EnableDefaultItems>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+  </PropertyGroup>
+  <ItemGroup>
+    {string.Join(Environment.NewLine, module.CodeFiles.Select(c =>
+  $"<Compile Include=\"{c}\" />"))}
+  </ItemGroup>
+  <ItemGroup>
+    {string.Join(Environment.NewLine,
+         new[] { Path.Combine(Path.GetDirectoryName(location), "CppSharp.Runtime.dll") }
+         .Union(module.Dependencies.Where(libraryMappings.ContainsKey).Select(d => libraryMappings[d]))
+         .Select(reference =>
+ $@"<Reference Include=""{Path.GetFileNameWithoutExtension(reference)}"">
+      <HintPath>{reference}</HintPath>
+    </Reference>"))}
+  </ItemGroup>
+</Project>".Trim());
+        }
+
+        private readonly Module module;
+        private readonly Dictionary<Module, string> libraryMappings;
+    }
+}

src/Generator/Types/Std/Stdlib.CSharp.cs

@@ -101,7 +101,7 @@ namespace CppSharp.Types.Std
             if (enconding == Encoding.ASCII)
                 return new CustomType("[MarshalAs(UnmanagedType.LPStr)] string");
             else if (enconding == Encoding.UTF8)
-                return new CustomType("[MarshalAs(UnmanagedType.LPUTF8Str)] string");
+                return new CustomType("[MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(CppSharp.Runtime.UTF8Marshaller))] string");
             else if (enconding == Encoding.Unicode || enconding == Encoding.BigEndianUnicode)
                 return new CustomType("[MarshalAs(UnmanagedType.LPWStr)] string");
             else if (enconding == Encoding.UTF32)

src/Package/CppSharp.Package.csproj

@@ -14,7 +14,6 @@ It can also be used as a library to parse native code into a syntax tree with a
   </PropertyGroup>
   
   <ItemGroup>
-    <PackageReference Include="System.CodeDom" />    
     <PackageReference Include="Microsoft.Win32.Registry" />
     <PackageReference Include="Microsoft.VisualStudio.Setup.Configuration.Interop" />
     <Content Include="$(PackageDir)runtimes\**" PackagePath="runtimes" />

src/Runtime/UTF8Marshaller.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace CppSharp.Runtime
+{
+    // HACK: .NET Standard 2.0 which we use in auto-building to support .NET Framework, lacks UnmanagedType.LPUTF8Str
+    public class UTF8Marshaller : ICustomMarshaler
+    {
+        public void CleanUpManagedData(object ManagedObj)
+        {
+        }
+
+        public void CleanUpNativeData(IntPtr pNativeData)
+            => Marshal.FreeHGlobal(pNativeData);
+
+        public int GetNativeDataSize() => -1;
+
+        public IntPtr MarshalManagedToNative(object managedObj)
+        {
+            if (managedObj == null)
+                return IntPtr.Zero;
+            if (!(managedObj is string))
+                throw new MarshalDirectiveException(
+                    "UTF8Marshaler must be used on a string.");
+
+            // not null terminated
+            byte[] strbuf = Encoding.UTF8.GetBytes((string) managedObj);
+            IntPtr buffer = Marshal.AllocHGlobal(strbuf.Length + 1);
+            Marshal.Copy(strbuf, 0, buffer, strbuf.Length);
+
+            // write the terminating null
+            Marshal.WriteByte(buffer + strbuf.Length, 0);
+            return buffer;
+        }
+
+        public unsafe object MarshalNativeToManaged(IntPtr str)
+        {
+            if (str == IntPtr.Zero)
+                return null;
+
+            int byteCount = 0;
+            var str8 = (byte*) str;
+            while (*(str8++) != 0) byteCount += sizeof(byte);
+
+            return Encoding.UTF8.GetString((byte*) str, byteCount);
+        }
+
+        public static ICustomMarshaler GetInstance(string pstrCookie)
+        {
+            if (marshaler == null)
+                marshaler = new UTF8Marshaller();
+            return marshaler;
+        }
+
+        private static UTF8Marshaller marshaler;
+    }
+}

tests/NamespacesBase/NamespacesBase.CSharp.csproj

@@ -1,12 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <TestGeneratorName>NamespacesDerived.Gen</TestGeneratorName>
-    <TestGeneratorProjectDir>$(MSBuildProjectDirectory)\..\NamespacesDerived\</TestGeneratorProjectDir>
-    <EnableGeneratorCompileItems>false</EnableGeneratorCompileItems>
-    <OutputPath>$(GenDir)NamespacesDerived</OutputPath>
-  </PropertyGroup>
-  
-  <ItemGroup>
-    <Compile Include="$(GenDir)NamespacesDerived\NamespacesBase.cs" />
-  </ItemGroup>
-</Project>

tests/NamespacesBase/premake4.lua

@@ -8,5 +8,4 @@ end
 
 group "Tests/Namespaces"
   SetupTestNativeProject("NamespacesBase")
-  targetdir (path.join(gendir, "NamespacesDerived"))
+  targetdir (path.join(gendir, "NamespacesDerived"))
-  SetupWrapper("NamespacesBase")

tests/NamespacesDerived/NamespacesDerived.CSharp.csproj

@@ -1,14 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <EnableGeneratorCompileItems>false</EnableGeneratorCompileItems>
-    <DocumentationFile>NamespacesDerived.CSharp.xml</DocumentationFile>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <Compile Include="$(GenDir)NamespacesDerived\NamespacesDerived.cs" />
-  </ItemGroup>
-  
-  <ItemGroup>
-    <ProjectReference Include="..\NamespacesBase\NamespacesBase.CSharp.csproj" />
-  </ItemGroup>
-</Project>

tests/NamespacesDerived/NamespacesDerived.Gen.cs

@@ -1,5 +1,4 @@
 using System.IO;
-using System.Reflection;
 using CppSharp.AST;
 using CppSharp.Generators;
 using CppSharp.Utils;
@@ -18,6 +17,7 @@ namespace CppSharp.Tests
             base.Setup(driver);
             driver.Options.GenerateDefaultValuesForArguments = true;
             driver.Options.GenerateClassTemplates = true;
+            driver.Options.CompileCode = true;
             driver.Options.DependentNameSpaces.Add("System.Runtime.CompilerServices");
             driver.Options.Modules[1].IncludeDirs.Add(GetTestsDirectory("NamespacesDerived"));
             var @base = "NamespacesBase";

tests/NamespacesDerived/NamespacesDerived.Tests.CSharp.csproj

@@ -1 +1,13 @@
-<Project Sdk="Microsoft.NET.Sdk" />
+<Project Sdk="Microsoft.NET.Sdk">
+  <ItemGroup>
+    <ProjectReference Include="NamespacesDerived.Gen.csproj" ReferenceOutputAssembly="false" />
+  </ItemGroup>
+  <ItemGroup>
+    <Reference Include="NamespacesBase">
+      <HintPath>..\..\build\gen\NamespacesDerived\NamespacesBase.dll</HintPath>
+    </Reference>
+    <Reference Include="NamespacesDerived">
+      <HintPath>..\..\build\gen\NamespacesDerived\NamespacesDerived.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+</Project>

tests/NamespacesDerived/premake4.lua

@@ -6,7 +6,4 @@ group "Tests/Namespaces"
   end
 
   SetupTestGeneratorProject("NamespacesDerived")
-  SetupTestProjectsCSharp("NamespacesDerived", "NamespacesBase")
+  SetupTestProjectsCSharp("NamespacesDerived", "NamespacesBase")
-
-  project("NamespacesDerived.Tests.CSharp")
-    links { "NamespacesBase.CSharp" }

tests/Test.props

@@ -9,7 +9,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="$(TestName).CSharp.csproj" Condition="$(MSBuildProjectName.EndsWith('CSharp'))" />
+    <ProjectReference Include="$(TestName).CSharp.csproj" Condition="$(MSBuildProjectName.EndsWith('CSharp')) AND $(TestName) != 'NamespacesDerived'" />
     <ProjectReference Include="$(NativeProjectsDir)$(TestName).Native.vcxproj" Condition="$(IsWindows)" ReferenceOutputAssembly="false" />
     <ProjectReference Include="$(NativeProjectsDir)$(TestName).CLI.vcxproj" Condition="$(MSBuildProjectName.EndsWith('CLI')) AND $(IsWindows)" />
   </ItemGroup>