Reprocess, load and hot swap changed assets using new asset pipeline

GlitchyEditor/src/AssetFile.bf

@@ -63,6 +63,8 @@ class AssetFile
 
 	public EditorContentManager ContentManager => _contentManager;
 
+	public bool UseNewAssetPipeline => _assetConfig?.ImporterConfig != null;
+
 	[AllowAppend]
 	public this(EditorContentManager contentManager, AssetNode assetNode)
 	{
@@ -82,24 +84,32 @@ class AssetFile
 	{
 		AssetFile assetFile = new AssetFile(contentManager, assetNode);
 
-		assetFile.LoadOrCreateAssetConfig();
+		assetFile.CheckForReprocessing();
-
-		// TODO: Remove this check once we only use the new processing pipeline
-		if (assetFile._assetConfig.ImporterConfig != null)
-		{
-			CachedAsset cacheEntry = assetFile._contentManager.AssetCache.GetCacheEntry(assetFile._assetConfig.AssetHandle);
-	
-			if (cacheEntry == null ||
-				cacheEntry.CreationTimestamp < assetFile._lastAssetEditTime ||
-				cacheEntry.CreationTimestamp < assetFile._lastConfigEditTime)
-			{
-				assetFile._contentManager.AssetConverter.QueueForProcessing(assetFile);
-			}
-		}
 
 		return assetFile;
 	}
 
+	public void CheckForReprocessing()
+	{
+		LoadOrCreateAssetConfig();
+
+		// TODO: Remove this check once we only use the new processing pipeline
+		if (!UseNewAssetPipeline)
+			return;
+
+		CachedAsset cacheEntry = _contentManager.AssetCache.GetCacheEntry(_assetConfig.AssetHandle);
+		
+		_lastAssetEditTime = File.GetLastWriteTimeUtc(_assetFile.Path);
+		_lastConfigEditTime = File.GetLastWriteTimeUtc(_assetConfigPath);
+
+		if (cacheEntry == null ||
+			cacheEntry.CreationTimestamp < _lastAssetEditTime ||
+			cacheEntry.CreationTimestamp < _lastConfigEditTime)
+		{
+			_contentManager.AssetConverter.QueueForProcessing(this);
+		}
+	}
+
 	/// Loads the asset config (.ass) file or creates it.
 	private void LoadOrCreateAssetConfig()
 	{
@@ -107,12 +117,12 @@ class AssetFile
 		{
 			LoadAssetConfig();
 		}
-		else
+		else if (_assetConfig == null)
 		{
 			CreateDefaultAssetLoader();
 		}
 
-		_lastAssetEditTime = File.GetLastWriteTimeUtc(_assetConfigPath);
+		_lastConfigEditTime = File.GetLastWriteTimeUtc(_assetConfigPath);
 	}
 
 	private void GenerateAssetHandle()
@@ -124,6 +134,8 @@ class AssetFile
 	{
 		String fileExtension = Path.GetExtension(_assetFile.Path, .. scope .());
 
+		Log.EngineLogger.Info($"Created config for {_assetFile.Path}");
+
 		_assetConfig = new AssetConfig();
 		
 		GenerateAssetHandle();
@@ -163,21 +175,33 @@ class AssetFile
 
 	private void LoadAssetConfig()
 	{
-		if (Bon.DeserializeFromFile(ref _assetConfig, _assetConfigPath) case .Err)
+		AssetConfig newAssetConfig = new AssetConfig();
+
+		if (Bon.DeserializeFromFile(ref newAssetConfig, _assetConfigPath) case .Err)
 		{
 			Log.EngineLogger.Error($"Failed to load asset config {_assetConfigPath}");
 
-			// TODO: Handle failure of asset config loading
+			delete newAssetConfig;
-			Runtime.NotImplemented();
+
+			return;
 		}
+
+		delete _assetConfig;
+		_assetConfig = newAssetConfig;
 	}
 
 	public void SaveAssetConfig()
 	{
-		gBonEnv.serializeFlags |= .Verbose;
+		//BonEnvironment bonEnv = scope .();
+
+		var oldFlags = gBonEnv.serializeFlags;
+
+		gBonEnv.serializeFlags |= .IncludeDefault | .Verbose;
 
 		Bon.SerializeIntoFile(_assetConfig, _assetConfigPath);
 
-		_assetConfig.Config.[Friend]_changed = false;
+		_assetConfig.Config?.[Friend]_changed = false;
+
+		gBonEnv.serializeFlags = oldFlags;
 	}
 }

GlitchyEditor/src/Assets/AssetCache.bf

@@ -35,14 +35,21 @@ class AssetCache
 	/// Current format version of loose asset file (.laf) file reader and writer.
 	public const uint16 FormatVersion = 1;
 
-	private append String _directory = .() ~ delete:append _;
+	private append String _directory = .();// ~ delete:append _;
 
-	private append Dictionary<AssetHandle, CachedAsset> _assets ~ delete:append _;
+	private append Dictionary<AssetHandle, CachedAsset> _assets = .();// ~ delete:append _;
 
 	public StringView CacheDirectory => _directory;
 
 	private bool _cacheLoaded;
 
+	private EditorContentManager _contentManager;
+
+	public this(EditorContentManager contentManager)
+	{
+		_contentManager = contentManager;
+	}
+
 	public ~this()
 	{
 		ClearCache();
@@ -198,7 +205,7 @@ class AssetCache
 
 		switch (asset.Compression)
 		{
-		case .L4Z:
+		case .LZ4:
 			uint maxCompressedSize = LZ4.LZ4F_CompressFrameBound((uint)data.Length);
 
 			uint8[] compressedData = new uint8[maxCompressedSize];
@@ -233,7 +240,7 @@ class AssetCache
 
 		Try!(writer.Write(dataToWrite));
 
-		// TODO: Notify content manager!
+		_contentManager.QueueAssetReload(asset.Handle);
 
 		return .Ok;
 	}
@@ -262,7 +269,7 @@ class AssetCache
 			FileStream fileStream = new FileStream();
 			Try!(OpenFileStream(asset, fileStream));
 			return fileStream;
-		case .L4Z:
+		case .LZ4:
 			FileStream compressedStream = scope FileStream();
 			Try!(OpenFileStream(asset, compressedStream));
 

GlitchyEditor/src/Assets/AssetHierarchy.bf

@@ -26,6 +26,8 @@ class AssetHierarchy
 	private append String _resourcesDirectory = .();
 	private append String _assetsDirectory = .();
 
+	private const float FileSystemDebounceSeconds = 0.5f;
+
 	private EditorContentManager _contentManager;
 
 	public TreeNode<AssetNode> RootNode => _assetRootNode;
@@ -257,21 +259,31 @@ class AssetHierarchy
 
 	private bool _fileTreeUpdateRequested = false;
 
+	private float _fileSystemDebounce;
+
 	/// Invokes the the file tree update on the mainthread.
 	private void DeferFileTreeUpdate()
 	{
+		_fileSystemDebounce = FileSystemDebounceSeconds;
+
 		if (!_fileTreeUpdateRequested)
 		{
 			_fileTreeUpdateRequested = true;
 
 			Application.Instance.InvokeOnMainThread(new () =>
 				{
+					_fileSystemDebounce -= Application.Instance.GameTime.DeltaTime;
+					
+					if (_fileSystemDebounce > 0)
+						return false;
+
 					UpdateFiles();
 					_fileTreeUpdateRequested = false;
+
+					return true;
 				});
 		}
 	}
-	
 	/// Initializes the FSW for the current ContentDirectory and registers the events.
 	private void SetupFileSystemWatcher()
 	{
@@ -518,7 +530,7 @@ class AssetHierarchy
 			fileName.RemoveFromEnd(AssetFile.ConfigFileExtension.Length);
 
 		String fileNameWithContentRoot = scope .();
-		Path.InternalCombine(fileNameWithContentRoot, _assetsDirectory, fileName);
+		Path.Combine(fileNameWithContentRoot, _assetsDirectory, fileName);
 
 		var nodeResult = GetNodeFromPath(fileNameWithContentRoot);
 
@@ -535,7 +547,21 @@ class AssetHierarchy
 		if (node->IsDirectory)
 			return;
 
+		if (!(node->AssetFile?.UseNewAssetPipeline ?? false))
 			OnFileContentChanged(node.Value);
+
+		_fileSystemDebounce = FileSystemDebounceSeconds;
+
+		Application.Instance.InvokeOnMainThread(new () => {
+			_fileSystemDebounce -= Application.Instance.GameTime.DeltaTime;
+			if (_fileSystemDebounce > 0)
+				return false;
+
+			node->AssetFile?.CheckForReprocessing();
+
+			return true;
+		});
+
 	}
 
 	private void FileRenamed(StringView oldFilePath, StringView newFilePath)
@@ -546,7 +572,6 @@ class AssetHierarchy
 		if (oldFilePath.EndsWith(AssetFile.ConfigFileExtension))
 			return;
 
-
 		String oldFileNameWithContentRoot = scope .();
 		Path.InternalCombine(oldFileNameWithContentRoot, _assetsDirectory, oldFilePath);
 		
@@ -560,7 +585,10 @@ class AssetHierarchy
 
 			if (File.Exists(oldConfigFileName) && !File.Exists(newConfigFileName))
 			{
-				if (File.Move(oldConfigFileName, newConfigFileName) case .Err(let value))
+				// TODO: Actually Move, but it's a bit complicated to manage cases, where external programs move the original file and
+				// rename an new file to the original name. So just copy the config to retain it.
+				// I don't think there is a clean way to solve this.
+				if (File.Copy(oldConfigFileName, newConfigFileName) case .Err(let value))
 				{
 					Log.EngineLogger.Error($"Failed to move file {oldConfigFileName} to {newConfigFileName}. Code: {value}");
 				}

GlitchyEditor/src/Assets/Importers/TextureImporter.bf

@@ -222,6 +222,7 @@ class TextureImporter : IAssetImporter
 	}
 }
 
+[BonTarget]
 enum GenerateMipMaps
 {
 	No,
@@ -717,7 +718,10 @@ class TextureLoader : IProcessedAssetLoader
 			Runtime.NotImplemented();
 		}
 
-		result.SamplerState = SamplerStateManager.GetSampler(sampler);
+		using (SamplerState samplerState = SamplerStateManager.GetSampler(sampler))
+		{
+			result.SamplerState = samplerState;
+		}
 
 		return result;
 	}

GlitchyEditor/src/Assets/TextureAssetLoader.bf

@@ -26,6 +26,8 @@ class TextureAssetPropertiesEditor : AssetPropertiesEditor
 
 	public override void ShowEditor()
 	{
+		return;
+
 		if (_textureConfig == null)
 			return;
 

GlitchyEditor/src/EditWindows/PropertiesWindow.bf

@@ -102,6 +102,8 @@ class PropertiesWindow : EditorWindow
 
 	private void ShowPropertiesEditor(AssetFile assetFile)
 	{
+		return;
+
 		if (_currentPropertiesEditor == null)
 			return;
 		

GlitchyEditor/src/EditorContentManager.bf

@@ -26,7 +26,7 @@ class EditorContentManager : IContentManager
 	private append Dictionary<AssetHandle, Asset> _handleToAsset = .();
 
 	private append AssetHierarchy _assetHierarchy = .(this);
-	private append AssetCache _assetCache = .() ~ delete:append _;
+	private append AssetCache _assetCache = .(this);
 	private append AssetConverter _assetConverter = .(this);
 
 	private append List<AssetHandle> _reloadQueue = .();
@@ -54,7 +54,16 @@ class EditorContentManager : IContentManager
 		if (assetNode.AssetFile?.LoadedAsset == null)
 			return;
 
-		_reloadQueue.Add(assetNode.AssetFile.LoadedAsset.Handle);
+		QueueAssetReload(assetNode.AssetFile.LoadedAsset.Handle);
+	}
+
+	public void QueueAssetReload(AssetHandle assetHandle)
+	{
+		// The asset isn't loaded, so we don't need to reload.
+		if (!_handleToAsset.ContainsKey(assetHandle))
+			return;
+
+		_reloadQueue.Add(assetHandle);
 	}
 
 	public void OnFileRenamed(AssetNode assetNode, StringView oldIdentifier)
@@ -373,29 +382,41 @@ class EditorContentManager : IContentManager
 
 		Log.EngineLogger.AssertDebug(oldAsset != null);
 
-		StringView oldIdentifier = oldAsset.Identifier;
-
-		GetResourceAndSubassetName(oldIdentifier, let resourceName, let subassetName);
-		
 		Result<TreeNode<AssetNode>> resultNode = AssetHierarchy.GetNodeFromAssetHandle(handle);
 
 		if (resultNode case .Err)
 		{
-			Log.EngineLogger.Error($"Could not find asset node for asset \"{oldIdentifier}\".");
+			Log.EngineLogger.Error($"Could not find asset node for asset \"{oldAsset.Identifier}\".");
 			return;
 		}
 
 		AssetNode assetNode = resultNode->Value;
 		AssetFile file = assetNode.AssetFile;
 
+		CachedAsset cacheEntry = _assetCache.GetCacheEntry(handle);
+
+		Asset loadedAsset;
+
+		// TODO: Remove this check once we no longer need the old stuff
+		if (cacheEntry != null)
+		{
+			loadedAsset = LoadFromCache(handle, false);
+		}
+		else
+		{
 			IAssetLoader assetLoader = GetAssetLoader(file);
 	
 			Stream stream = OpenStream(assetNode.Path, true);
 			
+			StringView oldIdentifier = oldAsset.Identifier;
+
+			GetResourceAndSubassetName(oldIdentifier, let resourceName, let subassetName);
+
 			// TODO: Add async loading!
-		Asset loadedAsset = assetLoader.LoadAsset(stream, file.AssetConfig.Config, resourceName, subassetName, this);
+			loadedAsset = assetLoader.LoadAsset(stream, file.AssetConfig.Config, resourceName, subassetName, this);
 		
 			delete stream;
+		}
 
 		if (loadedAsset == null)
 			return;
@@ -910,7 +931,7 @@ class EditorContentManager : IContentManager
 	}
 	
 	// TODO: obviously use a map or something...
-	TextureLoader textureLoader = new .();
+	TextureLoader textureLoader = new .() ~ delete _;
 
 	private IProcessedAssetLoader GetLoader(AssetType assetType)
 	{

GlitchyEngine/src/Application.bf

@@ -30,7 +30,7 @@ namespace GlitchyEngine
 		
 		private IContentManager _contentManager;
 
-		private append List<delegate void()> _jobQueue = .() ~ ClearAndDeleteItems!(_);
+		private append List<delegate bool()> _jobQueue = .() ~ ClearAndDeleteItems!(_);
 		private append Monitor _jobQueueMutex = .();
 
 		public bool IsRunning => _running;
@@ -233,7 +233,7 @@ namespace GlitchyEngine
 		}
 
 		/// Executes the given Job on the main thread. Takes ownership of the delegate.
-		public void InvokeOnMainThread(delegate void() ownJob)
+		public void InvokeOnMainThread(delegate bool() ownJob)
 		{
 			using (_jobQueueMutex.Enter())
 			{
@@ -249,10 +249,14 @@ namespace GlitchyEngine
 			{
 				for (let job in _jobQueue)
 				{
-					job();
+					if (job())
+					{
+						@job.RemoveFast();
+						delete job;
+					}
 				}
 
-				ClearAndDeleteItems!(_jobQueue);
+				//ClearAndDeleteItems!(_jobQueue);
 			}
 		}
 	}

GlitchyEngine/src/Content/AssetCompression.bf

@@ -1,7 +1,10 @@
+using Bon;
+
 namespace GlitchyEngine.Content;
 
+[BonTarget]
 enum AssetCompression : uint8
 {
 	None,
-	L4Z
+	LZ4
 }

GlitchyEngine/src/Scripting/ScriptEngine.bf

@@ -257,6 +257,8 @@ static class ScriptEngine
 					
 					_requestingReload = false;
 					_userAssemblyWatcher.StartRaisingEvents();
+
+					return true;
 				});
 	
 			});