Disable runtime marshalling

ScriptCore/ScriptGlue.cs

@@ -18,6 +18,12 @@ using System.Text;
 using System.Threading;
 using System.Xml.Linq;
 
+// This attribute is NOT optional, removing it silently breaks every engine function that takes or returns a bool!
+// With runtime marshalling enabled the CLR treats bool in an unmanaged function pointer as a 4-byte Win32 BOOL and checks all of eax,
+// but Beef only writes al (1 byte). The upper bytes are garbage, so the result is (almost) always true. Everything would still compile.
+// The price: in/ref/out are not allowed in unmanaged signatures of this assembly, which is why the generator emits raw pointers and pins with fixed.
+[assembly: DisableRuntimeMarshalling]
+
 namespace GlitchyEngine;
 
 /// <summary>
@@ -730,7 +736,7 @@ internal static unsafe partial class ScriptGlue
                     nativeString = new StringView(pointer, bytes);
                 }
                 
-                _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, fieldNameConverted, &nativeString, fullTypeNameConverted);
+                _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, &fieldNameConverted, &nativeString, &fullTypeNameConverted);
                 
                 if (onHeap)
                     StringView.FreeNativeMemory(nativeString);
@@ -750,7 +756,7 @@ internal static unsafe partial class ScriptGlue
                     // Skip Object Header (IntPtr) + Method Table (IntPtr) 
                     byte* valueRef = (byte*)*(IntPtr*)objectRef + sizeof(IntPtr);
                     float* fRef = (float*)(void*)valueRef;
-                    _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, fieldNameConverted, valueRef, fullTypeNameConverted);
+                    _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, &fieldNameConverted, valueRef, &fullTypeNameConverted);
                 }
                 break;
         }
@@ -782,7 +788,7 @@ internal static unsafe partial class ScriptGlue
 
         fixed (T* valuePtr = &value)
         {
-            _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, fieldNameConverted, valuePtr, fullTypeNameConverted);
+            _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, &fieldNameConverted, valuePtr, &fullTypeNameConverted);
         }
     }
 

ScriptCoreGenerator/ScriptGlueGenerator.cs

@@ -204,18 +204,6 @@ public class ScriptGlueGenerator : IIncrementalGenerator
             ReturnValueConversion = null
         });
 
-        // A bool in an unmanaged function pointer is marshalled as a 4-byte Win32 BOOL.
-        // Beef bools are 1 byte (only al is written), so we pass them as byte and convert manually.
-        beefTypeToMappedType.Add("bool", new MappedType
-        {
-            BeefTypeName = "bool",
-            CSharpTypeName = "byte",
-            CSharpWrapperType = "bool",
-            ReturnValueConversion = "return returnValue != 0;",
-            WrapperConvertInput = "byte {0} = (byte)({1} ? 1 : 0);",
-            WrapperOutConversion = "{1} = {0} != 0;"
-        });
-
         beefTypeToMappedType.Add("uint8*", new MappedType
         {
             BeefTypeName = "uint8*",
@@ -304,23 +292,13 @@ public class ScriptGlueGenerator : IIncrementalGenerator
             var (parameterType, parameterModifier) = DecodeType(param.Type, beefTypeToMappedType);
             parameters.Add(parameterType);
 
-            switch (parameterModifier)
-            {
-                case TypeModifier.In:
-                    parameterText.Append("in ");
-                    break;
-                case TypeModifier.Out:
-                    parameterText.Append("out ");
-                    break;
-                case TypeModifier.Ref:
-                    parameterText.Append("ref ");
-                    break;
-            }
-
             parameterText.Append(parameterType.CSharpTypeName);
 
             switch (parameterModifier)
             {
+                case TypeModifier.In:
+                case TypeModifier.Out:
+                case TypeModifier.Ref:
                 case TypeModifier.Pointer:
                     parameterText.Append("*");
                     break;
@@ -334,12 +312,11 @@ public class ScriptGlueGenerator : IIncrementalGenerator
 
     private static void GenerateWrapper(GlueMethod method, Dictionary<string, MappedType> beefTypeToMappedType, StringBuilder output)
     {
-        // MappedType returnType = DecodeType(method.ReturnType, beefTypeToMappedType);
-        //
         List<MappedType> parameters = new();
         StringBuilder parameterText = new();
-        StringBuilder parameterConversion = new();
+        List<string> parameterConversions = new();
-        StringBuilder cleanup = new();
+        List<string> cleanups = new();
+        List<string> afterReturnCleanups = new();
         StringBuilder call = new();
 
         var (returnType, _) = DecodeType(method.ReturnType, beefTypeToMappedType);
@@ -382,33 +359,36 @@ public class ScriptGlueGenerator : IIncrementalGenerator
 
             parameterText.Append($" {param.Name}");
 
-            switch (parameterModifier)
-            {
-                case TypeModifier.In:
-                    call.Append("in ");
-                    break;
-                case TypeModifier.Out:
-                    call.Append("out ");
-                    break;
-                case TypeModifier.Ref:
-                    call.Append("ref ");
-                    break;
-            }
-
             if (parameterModifier == TypeModifier.Out && parameterType.WrapperOutConversion is not null)
             {
                 string tmpArgName = $"{param.Name}Tmp";
 
-                call.Append($"var {tmpArgName}");
+                call.Append($"&{tmpArgName}");
 
+                parameterConversions.Add($"\t\t{parameterType.CSharpTypeName} {tmpArgName} = default;\n");
+
+                StringBuilder cleanup = new();
                 cleanup.Append("\t\t");
                 cleanup.AppendFormat(parameterType.WrapperOutConversion, tmpArgName, param.Name);
                 cleanup.AppendLine();
+                cleanups.Add(cleanup.ToString());
             }
             else if (parameterType.WrapperConvertInput is not null)
             {
+                switch (parameterModifier)
+                {
+                    case TypeModifier.In:
+                    case TypeModifier.Out:
+                    case TypeModifier.Ref:
+                    case TypeModifier.Pointer:
+                        call.Append("&");
+                        break;
+                }
+
                 string convertedParamName = $"{param.Name}Converted";
 
+                StringBuilder parameterConversion = new();
+                StringBuilder cleanup = new();
                 parameterConversion.AppendLine();
                 parameterConversion.Append("\t\t");
                 parameterConversion.AppendFormat(parameterType.WrapperConvertInput, convertedParamName, param.Name);
@@ -421,21 +401,40 @@ public class ScriptGlueGenerator : IIncrementalGenerator
                 }
 
                 call.Append(convertedParamName);
+                parameterConversions.Add(parameterConversion.ToString());
+                cleanups.Add(cleanup.ToString());
             }
             else
             {
-                call.Append(param.Name);
+                string paramName = param.Name;
+
+                switch (parameterModifier)
+                {
+                    case TypeModifier.In:
+                    case TypeModifier.Out:
+                    case TypeModifier.Ref:
+                        paramName = "__fixed" + param.Name;
+                        parameterConversions.Add($"\t\tfixed ({parameterType.CSharpTypeName}* {paramName} = &{param.Name}) {{");
+                        afterReturnCleanups.Add("\t\t}");
+                        break;
+                }
+
+                call.Append(paramName);
             }
         }
 
         call.Append(");");
 
+        cleanups.Reverse();
+
         output.AppendLine($$"""
                                 public static {{returnType.CSharpWrapperType}} {{method.Name}}({{parameterText}})
-                                {{{parameterConversion}}
+                                {
+                            {{string.Join("\n", parameterConversions)}}
                                     {{call}}
-                            {{cleanup}}
+                            {{string.Join("\n", cleanups)}}
                                     {{returnType.ReturnValueConversion}}
+                            {{string.Join("\n", afterReturnCleanups)}}
                                 }
 
                             """);