Add support for const char32_t* and fix const wchar_t* in linux / osx (#1444)

* Add support for const char32_t* and fix const wchar_t* in linux / osx

src/AST/TypeExtensions.cs

@@ -406,6 +406,7 @@
 
             return (pointee.IsPrimitiveType(PrimitiveType.Char) ||
                     pointee.IsPrimitiveType(PrimitiveType.Char16) ||
+                    pointee.IsPrimitiveType(PrimitiveType.Char32) ||
                     pointee.IsPrimitiveType(PrimitiveType.WideChar)) &&
                     pointer.QualifiedPointee.Qualifiers.IsConst;
         }

src/Generator/Types/Std/Stdlib.cs

@@ -155,49 +155,50 @@ namespace CppSharp.Types.Std
                 return new CustomType(typePrinter.IntPtrType);
             }
 
-            Type type = Type.Desugar();
+            var (enconding, _) = GetEncoding();
 
-            uint charWidth = 0;
-            if (type is PointerType pointerType)
-                charWidth = GetCharPtrWidth(pointerType);
-            else if (type.GetPointee()?.Desugar() is PointerType pointeePointerType)
-                charWidth = GetCharPtrWidth(pointeePointerType);
-
-            if (charWidth == 8)
-            { 
-                if (Context.Options.Encoding == Encoding.ASCII)
-                    return new CustomType("[MarshalAs(UnmanagedType.LPStr)] string");
+            if (enconding == Encoding.ASCII)
+                return new CustomType("[MarshalAs(UnmanagedType.LPStr)] string");
+            else if (enconding == Encoding.UTF8)
                 return new CustomType("[MarshalAs(UnmanagedType.LPUTF8Str)] string");
-            }
-
-            if (Context.Options.Encoding == Encoding.Unicode ||
-                Context.Options.Encoding == Encoding.BigEndianUnicode)
-            {
-                // NOTE: This will break if charWidth is not 16 bit which may
-                // happen if someone uses wchar_t on platforms where it's size is 32 bit.
-                // TODO: Create a custom marshaller to support that scenario.
-                // Once that's done consider supporting char32_t as well
+            else if (enconding == Encoding.Unicode || enconding == Encoding.BigEndianUnicode)
                 return new CustomType("[MarshalAs(UnmanagedType.LPWStr)] string");
-            }
-
-            if (Context.Options.Encoding == Encoding.UTF8 && charWidth == 16)
-                return new CustomType("[MarshalAs(UnmanagedType.LPWStr)] string"); // fallback;
+            else if (enconding == Encoding.UTF32)
+                return new CustomType("[MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(CppSharp.Runtime.UTF32Marshaller))] string");
 
             throw new System.NotSupportedException(
                 $"{Context.Options.Encoding.EncodingName} is not supported yet.");
         }
 
+        public uint GetCharWidth()
+        {
+            Type type = Type.Desugar();
+
+            if (type is PointerType pointerType)
+                return GetCharPtrWidth(pointerType);
+
+            if (type.GetPointee()?.Desugar() is PointerType pointeePointerType)
+                return GetCharPtrWidth(pointeePointerType);
+
+            return 0;
+        }
+
         public uint GetCharPtrWidth(PointerType pointer)
         {
             var pointee = pointer?.Pointee?.Desugar();
-            if (pointee != null)
+            if (pointee != null && pointee.IsPrimitiveType(out var primitiveType))
             {
-                if (pointee.IsPrimitiveType(PrimitiveType.Char))
-                    return Context.TargetInfo.CharWidth;
-                if (pointee.IsPrimitiveType(PrimitiveType.WideChar))
-                    return Context.TargetInfo.WCharWidth;
-                if (pointee.IsPrimitiveType(PrimitiveType.Char16))
-                    return Context.TargetInfo.Char16Width;
+                switch (primitiveType)
+                {
+                    case PrimitiveType.Char:
+                        return Context.TargetInfo.CharWidth;
+                    case PrimitiveType.WideChar:
+                        return Context.TargetInfo.WCharWidth;
+                    case PrimitiveType.Char16:
+                        return Context.TargetInfo.Char16Width;
+                    case PrimitiveType.Char32:
+                        return Context.TargetInfo.Char32Width;
+                }
             }
 
             return 0;
@@ -224,7 +225,7 @@ namespace CppSharp.Types.Std
             string bytes = $"__bytes{ctx.ParameterIndex}";
             string bytePtr = $"__bytePtr{ctx.ParameterIndex}";
             ctx.Before.WriteLine($@"byte[] {bytes} = global::System.Text.Encoding.{
-                GetEncodingClass(ctx.Parameter)}.GetBytes({param});");
+                GetEncoding().Name}.GetBytes({param});");
             ctx.Before.WriteLine($"fixed (byte* {bytePtr} = {bytes})");
             ctx.HasCodeBlock = true;
             ctx.Before.WriteOpenBraceAndIndent();
@@ -266,50 +267,45 @@ namespace CppSharp.Types.Std
                 textGenerator.WriteLine($"if ({ctx.ReturnVarName} == {nullPtr})");
                 textGenerator.WriteOpenBraceAndIndent();
                 textGenerator.WriteLine($"{ctx.Parameter.Name} = default({Type.Desugar()});");
-                textGenerator.WriteLine("return;");
+                textGenerator.WriteLine($"return{(ctx.Function.ReturnType.Type.IsPrimitiveType(PrimitiveType.Void) ? "" : " default")};");
                 textGenerator.UnindentAndWriteCloseBrace();
             }
 
-            string encoding = GetEncodingClass(ctx.Parameter);
+            string encoding = GetEncoding().Name;
             string type = GetTypeForCodePoint(encoding);
-            textGenerator.WriteLine($"var __retPtr = ({type}*) {returnVarName};");
-            textGenerator.WriteLine("int __length = 0;");
-            textGenerator.WriteLine($"while (*(__retPtr++) != 0) __length += sizeof({type});");
+            var retPtr = Generator.GeneratedIdentifier($"retPtr{ctx.ParameterIndex}");
+            var length = Generator.GeneratedIdentifier($"length{ctx.ParameterIndex}");
+            textGenerator.WriteLine($"var {retPtr} = ({type}*) {returnVarName};");
+            textGenerator.WriteLine($"int {length} = 0;");
+            textGenerator.WriteLine($"while (*({retPtr}++) != 0) {length} += sizeof({type});");
 
             ctx.Return.Write($@"global::System.Text.Encoding.{
-                encoding}.GetString((byte*) {returnVarName}, __length)");
+                encoding}.GetString((byte*) {returnVarName}, {length})");
         }
 
-        private string GetEncodingClass(Parameter parameter)
+        private (Encoding Encoding, string Name) GetEncoding()
         {
-            Type type = Type.Desugar();
-            Type pointee = type.GetPointee().Desugar();
-            var isChar = type.IsPointerToPrimitiveType(PrimitiveType.Char) ||
-                (pointee.IsPointerToPrimitiveType(PrimitiveType.Char) &&
-                 parameter != null &&
-                 (parameter.IsInOut || parameter.IsOut));
-
-            if (!isChar)
-                return (Context.TargetInfo.WCharWidth == 16) ?
-                    nameof(Encoding.Unicode) : nameof(Encoding.UTF32);
-
-            if (Context.Options.Encoding == Encoding.ASCII)
-                return nameof(Encoding.ASCII);
-
-            if (Context.Options.Encoding == Encoding.BigEndianUnicode)
-                return nameof(Encoding.BigEndianUnicode);
-
-            if (Context.Options.Encoding == Encoding.Unicode)
-                return nameof(Encoding.Unicode);
-
-            if (Context.Options.Encoding == Encoding.UTF32)
-                return nameof(Encoding.UTF32);
-
-            if (Context.Options.Encoding == Encoding.UTF7)
-                return nameof(Encoding.UTF7);
-
-            if (Context.Options.Encoding == Encoding.UTF8)
-                return nameof(Encoding.UTF8);
+            switch (GetCharWidth())
+            {
+                case 8:
+                    if (Context.Options.Encoding == Encoding.ASCII)
+                        return (Context.Options.Encoding, nameof(Encoding.ASCII));
+                    if (Context.Options.Encoding == Encoding.UTF8)
+                        return (Context.Options.Encoding, nameof(Encoding.UTF8));
+                    if (Context.Options.Encoding == Encoding.UTF7)
+                        return (Context.Options.Encoding, nameof(Encoding.UTF7));
+                    if (Context.Options.Encoding == Encoding.BigEndianUnicode)
+                        return (Context.Options.Encoding, nameof(Encoding.BigEndianUnicode));
+                    if (Context.Options.Encoding == Encoding.Unicode)
+                        return (Context.Options.Encoding, nameof(Encoding.Unicode));
+                    if (Context.Options.Encoding == Encoding.UTF32)
+                        return (Context.Options.Encoding, nameof(Encoding.UTF32));
+                    break;
+                case 16:
+                    return (Encoding.Unicode, nameof(Encoding.Unicode));
+                case 32:
+                    return (Encoding.UTF32, nameof(Encoding.UTF32));
+            }
 
             throw new System.NotSupportedException(
                 $"{Context.Options.Encoding.EncodingName} is not supported yet.");
@@ -348,6 +344,12 @@ namespace CppSharp.Types.Std
     {
     }
 
+    [TypeMap("const char32_t*", GeneratorKind = GeneratorKind.CSharp)]
+    [TypeMap("const char32_t*", GeneratorKind = GeneratorKind.CLI)]
+    public class ConstChar32TPointer : ConstCharPointer
+    {
+    }
+
     [TypeMap("basic_string<char, char_traits<char>, allocator<char>>", GeneratorKind = GeneratorKind.CSharp)]
     [TypeMap("basic_string<char, char_traits<char>, allocator<char>>", GeneratorKind = GeneratorKind.CLI)]
     public class String : TypeMap

src/Runtime/UTF32Marshaller.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace CppSharp.Runtime
+{
+    public class UTF32Marshaller : ICustomMarshaler
+    {
+        static private UTF32Marshaller marshaler;
+
+        public unsafe object MarshalNativeToManaged(IntPtr pNativeData)
+        {
+            var p = (Int32*)pNativeData;
+            int count = 0;
+            while (*p++ != 0)
+                ++count;
+            return Encoding.UTF32.GetString((byte*)pNativeData, count * 4);
+        }
+
+        public unsafe IntPtr MarshalManagedToNative(object ManagedObj)
+        {
+            if (!(ManagedObj is string @string))
+                return IntPtr.Zero;
+
+            fixed (char* stringPtr = @string)
+            {
+                var byteCount = Encoding.UTF32.GetByteCount(stringPtr, @string.Length);
+                var result = Marshal.AllocCoTaskMem(byteCount + 4);
+                Encoding.UTF32.GetBytes(stringPtr, @string.Length, (byte*)result, byteCount);
+                *(Int32*)(result + byteCount) = 0;
+                return result;
+            }
+        }
+
+        public void CleanUpNativeData(IntPtr pNativeData)
+        {
+            Marshal.FreeCoTaskMem(pNativeData);
+        }
+
+        public void CleanUpManagedData(object ManagedObj)
+        {
+        }
+
+        public int GetNativeDataSize()
+        {
+            return -1;
+        }
+
+        public static ICustomMarshaler GetInstance(string pstrCookie)
+        {
+            if (marshaler == null)
+                marshaler = new UTF32Marshaller();
+            return marshaler;
+        }
+    }
+}

tests/CSharp/CSharp.Tests.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Text;
 using CppSharp.Utils;
 using CSharp;
 using NUnit.Framework;
@@ -1485,7 +1486,42 @@ public unsafe class CSharpTests : GeneratorTestFixture
     [Test]
     public void TestStringMarshall()
     {
-        Assert.IsTrue(StringMarshall.CSharpString8(StringMarshall.CSharpString));
-        Assert.IsTrue(StringMarshall.CSharpString16(StringMarshall.CSharpString));
+        var strings = new[] { "ЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя" +
+                              "ѐёђѓєѕіїјљњћќѝўџѠѡѢѣѤѥѦѧѨѩѪѫѬѭѮѯѰѱѲѳѴѵѶѷѸѹѺѻѼѽѾѿҀҁҊҋҌҍҎҏҐґҒғҔҕҖҗҘҙҚқҜҝҞҟҠҡҢңҤҥҦҧҨҩ" +
+                              "ҪҫҬҭҮүҰұҲҳҴҵҶҷҸҹҺһҼҽҾҿӀӁӂӃӄӅӆӇӈӉӊӋӌӍӎӏӐӑӒӓӔӕӖӗӘәӚӛӜӝӞӟӠӡӢӣӤӥӦӧӨөӪӫӬӭӮӯӰӱӲӳӴӵӶӷӸӹӺӻӼӽ" +
+                              "ӾӿԀԁԂԃԄԅԆԇԈԉԊԋԌԍԎԏԐԑԒԓ",
+
+                              "აბგდევზთიკლმნოპჟრსტუფქღყშჩცძწჭხჯჰჱჲჳჴჵჶჷჸჹჺ",
+
+                              "ԱԲԳԴԵԶԷԸԹԺԻԼԽԾԿՀՁՂՃՄՅՆՇՈՉՊՋՌՍՎՏՐՑՒՓՔՕՖՙաբգդեզէըթժիլխծկհձղճմյնշոչպջռսվտրցւփքօֆև",
+
+                              "々〆〱〲〳〴〵〻〼ぁあぃいぅうぇえぉおかがきぎくぐけげこごさざしじすずせぜそぞただちぢっつづて" +
+                              "でとどなにぬねのはばぱひびぴふぶぷへべぺほぼぽまみむめもゃやゅゆょよらりるれろゎわゐゑをんゔゕ" +
+                              "ゖゝゞゟァアィイゥウェエォオカガキギクグケゲコゴサザシジスズセゼソゾタダチヂッツヅテデトドナニ" +
+                              "ヌネノハバパヒビピフブプヘベペホボポマミムメモャヤュユョヨラリルレロヮワヰヱヲンヴヵヶヷヸヹヺ" +
+                              "ーヽヾヿㇰㇱㇲㇳㇴㇵㇶㇷㇸㇹㇺㇻㇼㇽㇾㇿ",
+
+                              "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzªµºÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ" +
+                              "ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿĀāĂ㥹ĆćĈĉĊċČčĎďĐđĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħĨĩĪīĬĭĮįİıIJijĴĵ" +
+                              "ĶķĸĹĺĻļĽľĿŀŁłŃńŅņŇňʼnŊŋŌōŎŏŐőŒœŔŕŖŗŘřŚśŜŝŞşŠšŢţŤťŦŧŨũŪūŬŭŮůŰűŲųŴŵŶŷŸŹźŻżŽžſƀƁƂƃƄƅƆƇƈƉƊ" +
+                              "ƋƌƍƎƏƐƑƒƓƔƕƖƗƘƙƚƛƜƝƞƟƠơƢƣƤƥƦƧƨƩƪƫƬƭƮƯưƱƲƳƴƵƶƷƸƹƺƻƼƽƾƿǀǁǂǃDŽDždžLJLjljNJNjnjǍǎǏǐǑǒǓǔǕǖǗǘǙǚǛǜǝ" +
+                              "ǞǟǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯǰDZDzdzǴǵǶǷǸǹǺǻǼǽǾǿȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏȐȑȒȓȔȕȖȗȘșȚțȜȝȞȟȠȡȢȣȤȥȦȧȨȩȪȫȬȭȮȯȰȱȲȳ" +
+                              "ȴȵȶȷȸȹȺȻȼȽȾȿɀɁɂɃɄɅɆɇɈɉɊɋɌɍɎɏḀḁḂḃḄḅḆḇḈḉḊḋḌḍḎḏḐḑḒḓḔḕḖḗḘḙḚḛḜḝḞḟḠḡḢḣḤḥḦḧḨḩḪḫḬḭḮḯḰḱḲḳḴḵḶḷḸḹḺḻḼḽ" +
+                              "ḾḿṀṁṂṃṄṅṆṇṈṉṊṋṌṍṎṏṐṑṒṓṔṕṖṗṘṙṚṛṜṝṞṟṠṡṢṣṤṥṦṧṨṩṪṫṬṭṮṯṰṱṲṳṴṵṶṷṸṹṺṻṼṽṾṿẀẁẂẃẄẅẆẇẈẉẊẋẌẍẎẏẐẑẒẓẔẕẖẗẘẙẚ" +
+                              "ẛẞẠạẢảẤấẦầẨẩẪẫẬậẮắẰằẲẳẴẵẶặẸẹẺẻẼẽẾếỀềỂểỄễỆệỈỉỊịỌọỎỏỐốỒồỔổỖỗỘộỚớỜờỞởỠỡỢợỤụỦủỨứỪừỬửỮữỰựỲỳỴỵỶỷỸỹ" +
+                              "ⱠⱡⱢⱣⱤⱥⱦⱧⱨⱩⱪⱫⱬⱭⱱⱲⱳⱴⱵⱶⱷ" };
+
+        foreach (var @string in strings)
+        {
+            var cs = @string;
+            Assert.That(CSharp.CSharp.TestCSharpString(cs, out var @out), Is.EqualTo(cs));
+            Assert.That(@out, Is.EqualTo(cs));
+            Assert.That(CSharp.CSharp.TestCSharpStringWide(cs, out var outWide), Is.EqualTo(cs));
+            Assert.That(outWide, Is.EqualTo(cs));
+            Assert.That(CSharp.CSharp.TestCSharpString16(cs, out var out16), Is.EqualTo(cs));
+            Assert.That(out16, Is.EqualTo(cs));
+            Assert.That(CSharp.CSharp.TestCSharpString32(cs, out var out32), Is.EqualTo(cs));
+            Assert.That(out32, Is.EqualTo(cs));
+        }
     }
 }

tests/CSharp/CSharp.cpp

@@ -1779,12 +1779,34 @@ boolean_t takeTypemapTypedefParam(boolean_t b)
     return b;
 }
 
-bool StringMarshall::CSharpString8(const char* in)
+const char* TestCSharpString(const char* in, const char** out)
 {
-    return in[0] == 'C' && in[1] == '#';
+    static std::string ret;
+    ret = in;
+    *out = ret.data();
+    return ret.data();
 }
 
-bool StringMarshall::CSharpString16(const char16_t* in)
+const wchar_t* TestCSharpStringWide(const wchar_t* in, const wchar_t** out)
 {
-    return in[0] == 'C' && in[1] == '#';
+    static std::wstring ret;
+    ret = in;
+    *out = ret.data();
+    return ret.data();
+}
+
+const char16_t* TestCSharpString16(const char16_t* in, const char16_t** out)
+{
+    static std::u16string ret;
+    ret = in;
+    *out = ret.data();
+    return ret.data();
+}
+
+const char32_t* TestCSharpString32(const char32_t* in, const char32_t** out)
+{
+    static std::u32string ret;
+    ret = in;
+    *out = ret.data();
+    return ret.data();
 }

tests/CSharp/CSharp.h

@@ -1484,10 +1484,7 @@ struct TestVariableWithoutType
     static constexpr auto variable = create(n...);
 };
 
-class DLL_API StringMarshall
-{
-public:
-    static constexpr const char* CSharpString = "C#";
-    static bool CSharpString8(const char* in);
-    static bool CSharpString16(const char16_t* in);
-};
+DLL_API const char* TestCSharpString(const char* in, CS_OUT const char** out);
+DLL_API const wchar_t* TestCSharpStringWide(const wchar_t* in, CS_OUT const wchar_t** out);
+DLL_API const char16_t* TestCSharpString16(const char16_t* in, CS_OUT const char16_t** out);    
+DLL_API const char32_t* TestCSharpString32(const char32_t* in, CS_OUT const char32_t** out);