Start of Script Instance deserialization

- Also deserialize when stopping the game to restore pre play state
- Fixed ancient bug where ClearDictionaryAndDeleteKeys falsly deletes the dictionary
- Added ClearDictionaryAndDeleteValues helper mixin

GlitchyEditor/src/EditorLayer.bf

@@ -863,12 +863,14 @@ namespace GlitchyEditor
 			ImGui.FocusWindow(window);
 		}
 		
+		append Dictionary<UUID, SerializedObject> _prePlaySerializedData = .() ~ ClearDictionaryAndDeleteValues!(_prePlaySerializedData);
+
 		/// Starts the play mode for the current scene
 		private void OnScenePlay()
 		{
-			Dictionary<UUID, SerializedObject> serializedData = scope .();
+			Log.EngineLogger.AssertDebug(_prePlaySerializedData.Count == 0, "Somehow some entities are serialized.");
 
-			ScriptEngine.SerializeScriptInstances(serializedData);
+			ScriptEngine.SerializeScriptInstances(_prePlaySerializedData);
 			
 			_editor.SceneViewportWindow.EditorMode = false;
 			_sceneState = .Play;
@@ -881,18 +883,13 @@ namespace GlitchyEditor
 
 				SetActiveScene(runtimeScene, startRuntime: true, startSimulation: true);
 				
-				ScriptEngine.DeserializeScriptInstances(serializedData);
+				ScriptEngine.DeserializeScriptInstances(_prePlaySerializedData);
 			}
 
 			_editor.CurrentScene = _activeScene;
 
 			if (Application.Instance.Settings.EditorSettings.SwitchToPlayerOnPlay)
 				SwitchToPlayWindow();
-
-			for (let v in serializedData)
-			{
-				delete v.value;
-			}
 		}
 
 		/// Activates the given scene.
@@ -987,11 +984,24 @@ namespace GlitchyEditor
 			
 			if (Application.Instance.Settings.EditorSettings.SwitchToEditorOnStop)
 				SwitchToEditorWindow();
+
+			// Reconstruct state before play
+			ScriptEngine.DeserializeScriptInstances(_prePlaySerializedData);
+
+			ClearPrePlaySerializedData();
+		}
+
+		/// Clears the serialized data cache.
+		private void ClearPrePlaySerializedData()
+		{
+			ClearDictionaryAndDeleteValues!(_prePlaySerializedData);
 		}
 
 		/// Stops the scene and cleans up the subsystems to allow loading another scene.
 		private void CloseCurrentScene()
 		{
+			// Clear serialized data, so that we don't waste time deserializing it.
+			ClearPrePlaySerializedData();
 			OnSceneStop();
 
 			ScriptEngine.ClearEntityScriptFields();

GlitchyEngine/src/Helper.bf

@@ -67,7 +67,17 @@ namespace GlitchyEngine
 			{
 				for (var value in dictionary)
 					delete value.key;
-				delete dictionary;
+				dictionary.Clear();
+			}
+		}
+
+		public static mixin ClearDictionaryAndDeleteValues(var dictionary)
+		{
+			if (dictionary != null)
+			{
+				for (var value in dictionary)
+					delete value.value;
+				dictionary.Clear();
 			}
 		}
 

GlitchyEngine/src/Scripting/ScriptEngine.bf

@@ -960,11 +960,7 @@ static class ScriptEngine
 			if (script.ScriptClass.ClassName != "SerializationTest")
 				continue;
 
-			SerializedObject object = new SerializedObject(allObjects);
+			SerializedObject object = new SerializedObject(allObjects, script.EntityId);
-
-			object.Id = script.EntityId;
-			object.AllObjects = allObjects;
-
 			object.Serialize(script);
 		}
 	}

GlitchyEngine/src/Scripting/ScriptGlue.bf

@@ -560,6 +560,20 @@ static class ScriptGlue
 		newId = newObject.Id;
 	}
 	
+	[RegisterCall("ScriptGlue::Serialization_DeserializeField")]
+	public static void Serialization_DeserializeField(void* internalContext, SerializationType expectedType, MonoString* fieldName, uint8* target)
+	{
+		SerializedObject context = Internal.UnsafeCastToObject(internalContext) as SerializedObject;
+
+		Log.EngineLogger.AssertDebug(context != null);
+		
+		char8* name = Mono.mono_string_to_utf8(fieldName);
+
+		context.GetField(StringView(name), expectedType, target);
+
+		Mono.mono_free(name);
+	}
+
 #endregion
 
 	private static void RegisterCall<T>(String name, T method) where T : var

GlitchyEngine/src/Scripting/ScriptInstanceSerializer.bf

@@ -58,6 +58,10 @@ public enum SerializationType : int32
 			return 16;
 		case .EntityReference, .ComponentReference, .ObjectReference:
 			return sizeof(UUID);
+		case .String:
+			return sizeof(StringView);
+		case .Enum:
+			return sizeof(StringView);
 		default:
 			return 0;
 		}
@@ -74,16 +78,23 @@ class SerializedObject
 
 	public append Dictionary<StringView, (SerializationType PrimitiveType, uint8[16] Data)> Fields = .();
 
-	public this(Dictionary<UUID, SerializedObject> allObjects)
+	public this(Dictionary<UUID, SerializedObject> allObjects, UUID? id = null)
 	{
 		AllObjects = allObjects;
 
+		if (id == null)
+		{
 			// Make sure we have no duplicate keys
 			repeat
 			{
 				Id = UUID.Create();
 			}
 			while (AllObjects.ContainsKey(Id));
+		}
+		else
+		{
+			Id = id.Value;
+		}
 
 		AllObjects.Add(Id, this);
 	}
@@ -140,6 +151,33 @@ class SerializedObject
 		Fields.Add(nameCopy, (primitiveType, data));
 	}
 	
+	public void GetField(StringView fieldName, SerializationType expectedType, uint8* target)
+	{
+		if (!Fields.TryGetValue(fieldName, let field))
+			return;
+
+		if (expectedType != field.PrimitiveType)
+			return;
+
+		switch (field.PrimitiveType)
+		{
+		case .String, .Enum:
+#unwarn
+			StringView view = *(StringView*)&field.Data;
+
+			char8* stringPtr = view.Ptr;
+			int stringLen = view.Length;
+
+			// We just pass the raw utf8-Pointer and length to C#
+			Internal.MemCpy(target, &stringPtr, sizeof(void*));
+			Internal.MemCpy(target + 8, &stringLen, sizeof(int));
+		default:
+		// Most values can simply be copied, the conversion will be done in C#
+#unwarn
+		Internal.MemCpy(target, &field.Data, 16);
+		}
+	}
+	
 	typealias SerializeMethod = function MonoObject*(MonoObject* entity, void* contextPtr, MonoException** exception);
 	typealias DeserializeMethod = function MonoObject*(MonoObject* entity, void* contextPtr, MonoException** exception);
 
@@ -158,7 +196,7 @@ class SerializedObject
 
 	public void Deserialize(ScriptInstance scriptInstance)
 	{
-		DeserializeMethod deserialize = (DeserializeMethod)ScriptEngine.Classes.EntitySerializer.GetMethodThunk("Serialize", 2);
+		DeserializeMethod deserialize = (DeserializeMethod)ScriptEngine.Classes.EntitySerializer.GetMethodThunk("Deserialize", 2);
 
 		void* thisPtr = Internal.UnsafeCastToPtr(this);
 

ScriptCore/ScriptGlue.cs

@@ -145,5 +145,8 @@ internal static class ScriptGlue
     [MethodImpl(MethodImplOptions.InternalCall)]
     internal static extern void Serialization_CreateObject(IntPtr currentContext, out IntPtr context, out UUID id);
     
+    [MethodImpl(MethodImplOptions.InternalCall)]
+    public static extern unsafe void Serialization_DeserializeField(IntPtr internalContext, SerializationType expectedType, string fieldName, byte* value);
+
     #endregion
 }

ScriptCore/Serialization/EntitySerializer.cs

@@ -4,6 +4,8 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Text;
 using GlitchyEngine.Core;
 using GlitchyEngine.Extensions;
@@ -252,6 +254,343 @@ public static class EntitySerializer
         }
     }
     
+    public class DeserializationObject
+    {
+        private IntPtr _internalContext;
+
+        private UUID _id;
+
+        private Stack<string> _structScope = new();
+
+        private string _structScopeName;
+
+        public Dictionary<object, SerializedObject> SerializedClasses;
+
+        public DeserializationObject(IntPtr internalContext, UUID id, Dictionary<object, SerializedObject> serializedClasses)
+        {
+            _internalContext = internalContext;
+            _id = id;
+            SerializedClasses = serializedClasses;
+        }
+        
+        private (SerializedObject context, bool newContext) GetSerializedObject(object o)
+        {
+            SerializedObject context;
+
+            if (SerializedClasses.TryGetValue(o, out context))
+                return (context, false);
+            
+            ScriptGlue.Serialization_CreateObject(_internalContext, out IntPtr contextPtr, out UUID id);
+
+            context = new SerializedObject(contextPtr, id, SerializedClasses);
+
+            SerializedClasses.Add(o, context);
+
+            return (context, true);
+        }
+
+        private void PushScope(string name)
+        {
+            _structScope.Push(name);
+
+            _structScopeName += $"{name}.";
+        }
+
+        private void PopScope()
+        {
+            string scopeToRemove = _structScope.Pop();
+            _structScopeName.Remove(_structScopeName.Length - scopeToRemove.Length - 1);
+        }
+
+        private unsafe object GetFieldValue(string fieldName, SerializationType serializationType)
+        {
+            string completeFieldName = $"{_structScopeName}{fieldName}";
+
+            // Decimal is the larges primitive we store so we use a decimal as stack allocated memory (because stackalloc doesn't seem to work :(
+            //decimal backingFieldOnStack = 0.0m;
+            byte* rawData = stackalloc byte[16];
+            //byte* rawData = (byte*)&backingFieldOnStack;
+
+            ScriptGlue.Serialization_DeserializeField(_internalContext, serializationType, completeFieldName, rawData);
+
+            string GetString()
+            {
+                // rawData contains a Pointer and a string length!
+                byte* utf8Ptr = *(byte**)rawData;
+                ulong length = *(ulong*)(rawData + 8);
+
+                return Encoding.UTF8.GetString(utf8Ptr, (int)length);
+            }
+
+            switch (serializationType)
+            {
+                case SerializationType.Bool:
+                    return *(bool*)rawData;
+                case SerializationType.Char:
+                    return *(char*)rawData;
+                case SerializationType.String:
+                    return GetString();
+                case SerializationType.Int8:
+                    return *(sbyte*)rawData;
+                case SerializationType.Int16:
+                    return *(short*)rawData;
+                case SerializationType.Int32:
+                    return *(int*)rawData;
+                case SerializationType.Int64:
+                    return *(long*)rawData;
+                case SerializationType.UInt8:
+                    return *(byte*)rawData;
+                case SerializationType.UInt16:
+                    return *(ushort*)rawData;
+                case SerializationType.UInt32:
+                    return *(uint*)rawData;
+                case SerializationType.UInt64:
+                    return *(ulong*)rawData;
+	
+                case SerializationType.Float:
+                    return *(float*)rawData;
+                case SerializationType.Double:
+                    return *(double*)rawData;
+                case SerializationType.Decimal:
+                    return *(decimal*)rawData;
+
+                case SerializationType.Enum:
+                    string value = GetString();
+                    // TODO!
+                    return null;
+	
+                case SerializationType.EntityReference:
+                case SerializationType.ComponentReference:
+                case SerializationType.ObjectReference:
+                    return *(UUID*)rawData;
+                default:
+                    return null;
+            }
+        }
+
+        public void Deserialize(Entity entity)
+        {
+            DeserializeFields(entity);
+        }
+        
+        private bool DeserializeFields(object obj)
+        {
+            Type type = obj.GetType();
+
+            bool changed = false;
+
+            foreach (FieldInfo field in type.GetFields())
+            {
+                if (!EntitySerializer.SerializeField(field))
+                    continue;
+
+                changed |= DeserializeField(obj, field);
+            }
+
+            return changed;
+        }
+
+        private bool DeserializeField(object targetInstance, FieldInfo field)
+        {
+            Type fieldType = field.FieldType;
+
+            if (fieldType.IsPrimitive)
+            {
+                object value = DeserializePrimitive(field.Name, fieldType);
+
+                if (value != null)
+                {
+                    field.SetValue(targetInstance, value);
+                    return true;
+                }
+            }
+            else if (fieldType == typeof(string))
+            {
+                object value = GetFieldValue(field.Name, SerializationType.String);
+
+                if (value != null)
+                {
+                    field.SetValue(targetInstance, value);
+                    return true;
+                }
+            }
+            else if (fieldType.IsEnum)
+            {
+            //    SerializeEnum(fieldName, fieldValue, fieldType);
+            }
+            else if (fieldType.IsArray)
+            {
+            //    Log.Error($"Array serialization not yet implemented");
+            }
+            else if (fieldType.IsGenericType)
+            {
+                if (fieldType.GetGenericTypeDefinition() == typeof(List<>))
+                {
+                    //SerializeList(fieldName, type, o);
+                    Log.Error($"List serialization not yet implemented");
+                }
+                 //else if (fieldType.GetGenericTypeDefinition() == typeof(Dictionary<,>))
+                //{
+                //    ImGui.Text($"{fieldName} Dictionary");
+                //}
+                else
+                {
+                    // TODO: what to do?
+                    Log.Error($"Generic class serialization not yet implemented");
+                }
+            }
+            else if (fieldType.IsValueType)
+            {
+                object structValue = field.GetValue(targetInstance);
+
+                bool changed = DeserializeStruct(field.Name, structValue);
+
+                if (changed)
+                {
+                    field.SetValue(targetInstance, structValue);
+                    return true;
+                }
+            }
+            else if (fieldType.IsClass)
+            {
+            //    SerializeClass(fieldName, fieldValue, fieldType);
+            }
+            else
+            {
+                Log.Error($"Encountered unhandled type \"{fieldType}\" while serializing.");
+            }
+
+            return false;
+        }
+
+        private object DeserializePrimitive(string fieldName, Type fieldType)
+        {
+            Debug.Assert(fieldType.IsPrimitive, $"{fieldType} is not a primitive type.");
+
+            SerializationType expectedType = SerializationType.None;
+            
+            if (fieldType == typeof(bool))
+                expectedType = SerializationType.Bool;
+            else if (fieldType == typeof(char))
+                expectedType = SerializationType.Char;
+            else if (fieldType == typeof(byte))
+                expectedType = SerializationType.UInt8;
+            else if (fieldType == typeof(sbyte))
+                expectedType = SerializationType.Int8;
+            else if (fieldType == typeof(ushort))
+                expectedType = SerializationType.UInt16;
+            else if (fieldType == typeof(short))
+                expectedType = SerializationType.Int16;
+            else if (fieldType == typeof(uint))
+                expectedType = SerializationType.UInt32;
+            else if (fieldType == typeof(int))
+                expectedType = SerializationType.Int32;
+            else if (fieldType == typeof(ulong))
+                expectedType = SerializationType.UInt64;
+            else if (fieldType == typeof(long))
+                expectedType = SerializationType.Int64;
+            else if (fieldType == typeof(float))
+                expectedType = SerializationType.Float;
+            else if (fieldType == typeof(double))
+                expectedType = SerializationType.Double;
+            else if (fieldType == typeof(decimal))
+                expectedType = SerializationType.Decimal;
+            else
+            {
+                Log.Error($"The primitive {fieldType} is not implemented.");
+            }
+
+            return GetFieldValue(fieldName, expectedType);
+        }
+
+        private void SerializeEnum(string fieldName, object fieldValue, Type fieldType)
+        {
+            //AddField(fieldName, SerializationType.Enum, fieldValue.ToString());
+        }
+
+        private bool DeserializeStruct(string fieldName, object targetInstance)
+        {
+            PushScope(fieldName);
+
+            bool changed = DeserializeFields(targetInstance);
+
+            PopScope();
+
+            return changed;
+        }
+
+        private void SerializeClass(string fieldName, object fieldValue, Type fieldType)
+        {
+            //if (typeof(Entity).IsAssignableFrom(fieldType))
+            //{
+            //    AddField(fieldName, SerializationType.EntityReference, ((Entity)fieldValue)?.UUID ?? UUID.Zero);
+            //}
+            //else if (typeof(Component).IsAssignableFrom(fieldType))
+            //{
+            //        AddField(fieldName, SerializationType.ComponentReference, ((Component)fieldValue)?.UUID ?? UUID.Zero);
+            //}
+            //else
+            //{
+            //    if (fieldValue == null)
+            //    {
+            //        AddField(fieldName, SerializationType.ObjectReference, UUID.Zero);
+            //    }
+            //    else
+            //    {
+            //        var (context, newContext) = GetSerializedObject(fieldValue);
+
+            //        if (newContext)
+            //        {
+            //            context.SerializeFields(fieldValue);
+            //        }
+
+            //        AddField(fieldName, SerializationType.ObjectReference, context._id);   
+            //    }
+            //}
+        }
+    }
+
+
+    public static void Serialize(Entity entity, IntPtr internalContext)
+    {
+        SerializedObject obj = new SerializedObject(internalContext, UUID.Zero, new Dictionary<object, SerializedObject>());
+        obj.Serialize(entity);
+    }
+    
+    public static void Deserialize(Entity entity, IntPtr internalContext)
+    {
+        DeserializationObject obj = new DeserializationObject(internalContext, UUID.Zero, new Dictionary<object, SerializedObject>());
+        obj.Deserialize(entity);
+    }
+
+    public static bool SerializeField(FieldInfo fieldInfo)
+    {
+        var serializeField = fieldInfo.HasCustomAttribute<SerializeFieldAttribute>();
+        var dontSerializeField = fieldInfo.HasCustomAttribute<DontSerializeFieldAttribute>();
+        //var hideField = fieldInfo.HasCustomAttribute<HideInEditorAttribute>();
+        var showField = fieldInfo.HasCustomAttribute<ShowInEditorAttribute>();
+
+        if (fieldInfo.IsPublic)
+        {
+            if (dontSerializeField)
+                return false;
+
+            return true;
+        }
+        
+        if (serializeField)
+            return true;
+        
+        if (showField && !dontSerializeField)
+            return true;
+
+        return false;
+    }
+}
+
+/*
+ *
+ *
    public class SerializationContext
    {
    StringBuilder _builder = new();
@@ -496,40 +835,5 @@ public static class EntitySerializer
    }
    }
    
-    //public static void Serialize(Entity entity, SerializationContext context)
+ *
-    //{
+ */
-    //    context ??= new SerializationContext();
-        
-    //    context.Serialize($"Entity: {entity.UUID}", typeof(Entity), entity, false);
-    //}
-
-    public static void Serialize(Entity entity, IntPtr internalContext)
-    {
-        SerializedObject obj = new SerializedObject(internalContext, UUID.Zero, new Dictionary<object, SerializedObject>());
-        obj.Serialize(entity);
-    }
-
-    public static bool SerializeField(FieldInfo fieldInfo)
-    {
-        var serializeField = fieldInfo.HasCustomAttribute<SerializeFieldAttribute>();
-        var dontSerializeField = fieldInfo.HasCustomAttribute<DontSerializeFieldAttribute>();
-        //var hideField = fieldInfo.HasCustomAttribute<HideInEditorAttribute>();
-        var showField = fieldInfo.HasCustomAttribute<ShowInEditorAttribute>();
-
-        if (fieldInfo.IsPublic)
-        {
-            if (dontSerializeField)
-                return false;
-
-            return true;
-        }
-        
-        if (serializeField)
-            return true;
-        
-        if (showField && !dontSerializeField)
-            return true;
-
-        return false;
-    }
-}