Disable runtime marshalling

2026-11-04 12:51:48 +00:00 · 2026-09-19 23:43:25 +02:00
parent da349465fd
commit 14c1568e98
2 changed files with 96 additions and 91 deletions
@@ -18,6 +18,12 @@ using System.Text;
 using System.Threading;
 using System.Xml.Linq;

+// This attribute is NOT optional, removing it silently breaks every engine function that takes or returns a bool!
+// With runtime marshalling enabled the CLR treats bool in an unmanaged function pointer as a 4-byte Win32 BOOL and checks all of eax,
+// but Beef only writes al (1 byte). The upper bytes are garbage, so the result is (almost) always true. Everything would still compile.
+// The price: in/ref/out are not allowed in unmanaged signatures of this assembly, which is why the generator emits raw pointers and pins with fixed.
+[assembly: DisableRuntimeMarshalling]
+
 namespace GlitchyEngine;

 /// <summary>
@@ -730,7 +736,7 @@ internal static unsafe partial class ScriptGlue
                    nativeString = new StringView(pointer, bytes);
                }
                
-                _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, fieldNameConverted, &nativeString, fullTypeNameConverted);
+                _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, &fieldNameConverted, &nativeString, &fullTypeNameConverted);
                
                if (onHeap)
                    StringView.FreeNativeMemory(nativeString);
@@ -750,7 +756,7 @@ internal static unsafe partial class ScriptGlue
                    // Skip Object Header (IntPtr) + Method Table (IntPtr) 
                    byte* valueRef = (byte*)*(IntPtr*)objectRef + sizeof(IntPtr);
                    float* fRef = (float*)(void*)valueRef;
-                    _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, fieldNameConverted, valueRef, fullTypeNameConverted);
+                    _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, &fieldNameConverted, valueRef, &fullTypeNameConverted);
                }
                break;
        }
@@ -782,7 +788,7 @@ internal static unsafe partial class ScriptGlue

        fixed (T* valuePtr = &value)
        {
-            _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, fieldNameConverted, valuePtr, fullTypeNameConverted);
+            _engineFunctions.Serialization_SerializeField((void*)serializationContext, type, &fieldNameConverted, valuePtr, &fullTypeNameConverted);
        }
    }

@@ -204,18 +204,6 @@ public class ScriptGlueGenerator : IIncrementalGenerator
            ReturnValueConversion = null
        });

-        // A bool in an unmanaged function pointer is marshalled as a 4-byte Win32 BOOL.
-        // Beef bools are 1 byte (only al is written), so we pass them as byte and convert manually.
-        beefTypeToMappedType.Add("bool", new MappedType
-        {
-            BeefTypeName = "bool",
-            CSharpTypeName = "byte",
-            CSharpWrapperType = "bool",
-            ReturnValueConversion = "return returnValue != 0;",
-            WrapperConvertInput = "byte {0} = (byte)({1} ? 1 : 0);",
-            WrapperOutConversion = "{1} = {0} != 0;"
-        });
-
        beefTypeToMappedType.Add("uint8*", new MappedType
        {
            BeefTypeName = "uint8*",
@@ -304,23 +292,13 @@ public class ScriptGlueGenerator : IIncrementalGenerator
            var (parameterType, parameterModifier) = DecodeType(param.Type, beefTypeToMappedType);
            parameters.Add(parameterType);

-            switch (parameterModifier)
-            {
-                case TypeModifier.In:
-                    parameterText.Append("in ");
-                    break;
-                case TypeModifier.Out:
-                    parameterText.Append("out ");
-                    break;
-                case TypeModifier.Ref:
-                    parameterText.Append("ref ");
-                    break;
-            }
-
            parameterText.Append(parameterType.CSharpTypeName);

            switch (parameterModifier)
            {
+                case TypeModifier.In:
+                case TypeModifier.Out:
+                case TypeModifier.Ref:
                case TypeModifier.Pointer:
                    parameterText.Append("*");
                    break;
@@ -334,12 +312,11 @@ public class ScriptGlueGenerator : IIncrementalGenerator

    private static void GenerateWrapper(GlueMethod method, Dictionary<string, MappedType> beefTypeToMappedType, StringBuilder output)
    {
-        // MappedType returnType = DecodeType(method.ReturnType, beefTypeToMappedType);
-        //
        List<MappedType> parameters = new();
        StringBuilder parameterText = new();
-        StringBuilder parameterConversion = new();
-        StringBuilder cleanup = new();
+        List<string> parameterConversions = new();
+        List<string> cleanups = new();
+        List<string> afterReturnCleanups = new();
        StringBuilder call = new();

        var (returnType, _) = DecodeType(method.ReturnType, beefTypeToMappedType);
@@ -382,33 +359,36 @@ public class ScriptGlueGenerator : IIncrementalGenerator

            parameterText.Append($" {param.Name}");

-            switch (parameterModifier)
-            {
-                case TypeModifier.In:
-                    call.Append("in ");
-                    break;
-                case TypeModifier.Out:
-                    call.Append("out ");
-                    break;
-                case TypeModifier.Ref:
-                    call.Append("ref ");
-                    break;
-            }
-
            if (parameterModifier == TypeModifier.Out && parameterType.WrapperOutConversion is not null)
            {
                string tmpArgName = $"{param.Name}Tmp";

-                call.Append($"var {tmpArgName}");
+                call.Append($"&{tmpArgName}");

+                parameterConversions.Add($"\t\t{parameterType.CSharpTypeName} {tmpArgName} = default;\n");
+
+                StringBuilder cleanup = new();
                cleanup.Append("\t\t");
                cleanup.AppendFormat(parameterType.WrapperOutConversion, tmpArgName, param.Name);
                cleanup.AppendLine();
+                cleanups.Add(cleanup.ToString());
            }
            else if (parameterType.WrapperConvertInput is not null)
            {
+                switch (parameterModifier)
+                {
+                    case TypeModifier.In:
+                    case TypeModifier.Out:
+                    case TypeModifier.Ref:
+                    case TypeModifier.Pointer:
+                        call.Append("&");
+                        break;
+                }
+
                string convertedParamName = $"{param.Name}Converted";

+                StringBuilder parameterConversion = new();
+                StringBuilder cleanup = new();
                parameterConversion.AppendLine();
                parameterConversion.Append("\t\t");
                parameterConversion.AppendFormat(parameterType.WrapperConvertInput, convertedParamName, param.Name);
@@ -421,21 +401,40 @@ public class ScriptGlueGenerator : IIncrementalGenerator
                }

                call.Append(convertedParamName);
+                parameterConversions.Add(parameterConversion.ToString());
+                cleanups.Add(cleanup.ToString());
            }
            else
            {
-                call.Append(param.Name);
+                string paramName = param.Name;
+
+                switch (parameterModifier)
+                {
+                    case TypeModifier.In:
+                    case TypeModifier.Out:
+                    case TypeModifier.Ref:
+                        paramName = "__fixed" + param.Name;
+                        parameterConversions.Add($"\t\tfixed ({parameterType.CSharpTypeName}* {paramName} = &{param.Name}) {{");
+                        afterReturnCleanups.Add("\t\t}");
+                        break;
+                }
+
+                call.Append(paramName);
            }
        }

        call.Append(");");

+        cleanups.Reverse();
+
        output.AppendLine($$"""
                                public static {{returnType.CSharpWrapperType}} {{method.Name}}({{parameterText}})
-                                {{{parameterConversion}}
+                                {
+                            {{string.Join("\n", parameterConversions)}}
                                    {{call}}
-                            {{cleanup}}
+                            {{string.Join("\n", cleanups)}}
                                    {{returnType.ReturnValueConversion}}
+                            {{string.Join("\n", afterReturnCleanups)}}
                                }

                            """);