From b2806d9460c2f93c7a7de69a918f03294050f3df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20L=C3=BCbe=C3=9F?= Date: Sun, 2 Jun 2024 00:20:32 +0200 Subject: [PATCH] Reprocess, load and hot swap changed assets using new asset pipeline --- GlitchyEditor/src/AssetFile.bf | 68 +++++++++++++------ GlitchyEditor/src/Assets/AssetCache.bf | 17 +++-- GlitchyEditor/src/Assets/AssetHierarchy.bf | 38 +++++++++-- .../src/Assets/Importers/TextureImporter.bf | 6 +- .../src/Assets/TextureAssetLoader.bf | 2 + .../src/EditWindows/PropertiesWindow.bf | 2 + GlitchyEditor/src/EditorContentManager.bf | 57 +++++++++++----- GlitchyEngine/src/Application.bf | 12 ++-- GlitchyEngine/src/Content/AssetCompression.bf | 5 +- GlitchyEngine/src/Scripting/ScriptEngine.bf | 2 + 10 files changed, 153 insertions(+), 56 deletions(-) diff --git a/GlitchyEditor/src/AssetFile.bf b/GlitchyEditor/src/AssetFile.bf index b5867f3..4dacea6 100644 --- a/GlitchyEditor/src/AssetFile.bf +++ b/GlitchyEditor/src/AssetFile.bf @@ -63,6 +63,8 @@ class AssetFile public EditorContentManager ContentManager => _contentManager; + public bool UseNewAssetPipeline => _assetConfig?.ImporterConfig != null; + [AllowAppend] public this(EditorContentManager contentManager, AssetNode assetNode) { @@ -82,24 +84,32 @@ class AssetFile { AssetFile assetFile = new AssetFile(contentManager, assetNode); - assetFile.LoadOrCreateAssetConfig(); - - // TODO: Remove this check once we only use the new processing pipeline - if (assetFile._assetConfig.ImporterConfig != null) - { - CachedAsset cacheEntry = assetFile._contentManager.AssetCache.GetCacheEntry(assetFile._assetConfig.AssetHandle); - - if (cacheEntry == null || - cacheEntry.CreationTimestamp < assetFile._lastAssetEditTime || - cacheEntry.CreationTimestamp < assetFile._lastConfigEditTime) - { - assetFile._contentManager.AssetConverter.QueueForProcessing(assetFile); - } - } + assetFile.CheckForReprocessing(); return assetFile; } + public void CheckForReprocessing() + { + LoadOrCreateAssetConfig(); + + // TODO: Remove this check once we only use the new processing pipeline + if (!UseNewAssetPipeline) + return; + + CachedAsset cacheEntry = _contentManager.AssetCache.GetCacheEntry(_assetConfig.AssetHandle); + + _lastAssetEditTime = File.GetLastWriteTimeUtc(_assetFile.Path); + _lastConfigEditTime = File.GetLastWriteTimeUtc(_assetConfigPath); + + if (cacheEntry == null || + cacheEntry.CreationTimestamp < _lastAssetEditTime || + cacheEntry.CreationTimestamp < _lastConfigEditTime) + { + _contentManager.AssetConverter.QueueForProcessing(this); + } + } + /// Loads the asset config (.ass) file or creates it. private void LoadOrCreateAssetConfig() { @@ -107,12 +117,12 @@ class AssetFile { LoadAssetConfig(); } - else + else if (_assetConfig == null) { CreateDefaultAssetLoader(); } - _lastAssetEditTime = File.GetLastWriteTimeUtc(_assetConfigPath); + _lastConfigEditTime = File.GetLastWriteTimeUtc(_assetConfigPath); } private void GenerateAssetHandle() @@ -123,7 +133,9 @@ class AssetFile private void CreateDefaultAssetLoader() { String fileExtension = Path.GetExtension(_assetFile.Path, .. scope .()); - + + Log.EngineLogger.Info($"Created config for {_assetFile.Path}"); + _assetConfig = new AssetConfig(); GenerateAssetHandle(); @@ -163,21 +175,33 @@ class AssetFile private void LoadAssetConfig() { - if (Bon.DeserializeFromFile(ref _assetConfig, _assetConfigPath) case .Err) + AssetConfig newAssetConfig = new AssetConfig(); + + if (Bon.DeserializeFromFile(ref newAssetConfig, _assetConfigPath) case .Err) { Log.EngineLogger.Error($"Failed to load asset config {_assetConfigPath}"); - // TODO: Handle failure of asset config loading - Runtime.NotImplemented(); + delete newAssetConfig; + + return; } + + delete _assetConfig; + _assetConfig = newAssetConfig; } public void SaveAssetConfig() { - gBonEnv.serializeFlags |= .Verbose; + //BonEnvironment bonEnv = scope .(); + + var oldFlags = gBonEnv.serializeFlags; + + gBonEnv.serializeFlags |= .IncludeDefault | .Verbose; Bon.SerializeIntoFile(_assetConfig, _assetConfigPath); - _assetConfig.Config.[Friend]_changed = false; + _assetConfig.Config?.[Friend]_changed = false; + + gBonEnv.serializeFlags = oldFlags; } } \ No newline at end of file diff --git a/GlitchyEditor/src/Assets/AssetCache.bf b/GlitchyEditor/src/Assets/AssetCache.bf index db27be0..82f219d 100644 --- a/GlitchyEditor/src/Assets/AssetCache.bf +++ b/GlitchyEditor/src/Assets/AssetCache.bf @@ -35,14 +35,21 @@ class AssetCache /// Current format version of loose asset file (.laf) file reader and writer. public const uint16 FormatVersion = 1; - private append String _directory = .() ~ delete:append _; + private append String _directory = .();// ~ delete:append _; - private append Dictionary _assets ~ delete:append _; + private append Dictionary _assets = .();// ~ delete:append _; public StringView CacheDirectory => _directory; private bool _cacheLoaded; + private EditorContentManager _contentManager; + + public this(EditorContentManager contentManager) + { + _contentManager = contentManager; + } + public ~this() { ClearCache(); @@ -198,7 +205,7 @@ class AssetCache switch (asset.Compression) { - case .L4Z: + case .LZ4: uint maxCompressedSize = LZ4.LZ4F_CompressFrameBound((uint)data.Length); uint8[] compressedData = new uint8[maxCompressedSize]; @@ -233,7 +240,7 @@ class AssetCache Try!(writer.Write(dataToWrite)); - // TODO: Notify content manager! + _contentManager.QueueAssetReload(asset.Handle); return .Ok; } @@ -262,7 +269,7 @@ class AssetCache FileStream fileStream = new FileStream(); Try!(OpenFileStream(asset, fileStream)); return fileStream; - case .L4Z: + case .LZ4: FileStream compressedStream = scope FileStream(); Try!(OpenFileStream(asset, compressedStream)); diff --git a/GlitchyEditor/src/Assets/AssetHierarchy.bf b/GlitchyEditor/src/Assets/AssetHierarchy.bf index dfd085f..f092339 100644 --- a/GlitchyEditor/src/Assets/AssetHierarchy.bf +++ b/GlitchyEditor/src/Assets/AssetHierarchy.bf @@ -26,6 +26,8 @@ class AssetHierarchy private append String _resourcesDirectory = .(); private append String _assetsDirectory = .(); + private const float FileSystemDebounceSeconds = 0.5f; + private EditorContentManager _contentManager; public TreeNode RootNode => _assetRootNode; @@ -257,21 +259,31 @@ class AssetHierarchy private bool _fileTreeUpdateRequested = false; + private float _fileSystemDebounce; + /// Invokes the the file tree update on the mainthread. private void DeferFileTreeUpdate() { + _fileSystemDebounce = FileSystemDebounceSeconds; + if (!_fileTreeUpdateRequested) { _fileTreeUpdateRequested = true; Application.Instance.InvokeOnMainThread(new () => { + _fileSystemDebounce -= Application.Instance.GameTime.DeltaTime; + + if (_fileSystemDebounce > 0) + return false; + UpdateFiles(); _fileTreeUpdateRequested = false; + + return true; }); } } - /// Initializes the FSW for the current ContentDirectory and registers the events. private void SetupFileSystemWatcher() { @@ -518,7 +530,7 @@ class AssetHierarchy fileName.RemoveFromEnd(AssetFile.ConfigFileExtension.Length); String fileNameWithContentRoot = scope .(); - Path.InternalCombine(fileNameWithContentRoot, _assetsDirectory, fileName); + Path.Combine(fileNameWithContentRoot, _assetsDirectory, fileName); var nodeResult = GetNodeFromPath(fileNameWithContentRoot); @@ -535,7 +547,21 @@ class AssetHierarchy if (node->IsDirectory) return; - OnFileContentChanged(node.Value); + if (!(node->AssetFile?.UseNewAssetPipeline ?? false)) + OnFileContentChanged(node.Value); + + _fileSystemDebounce = FileSystemDebounceSeconds; + + Application.Instance.InvokeOnMainThread(new () => { + _fileSystemDebounce -= Application.Instance.GameTime.DeltaTime; + if (_fileSystemDebounce > 0) + return false; + + node->AssetFile?.CheckForReprocessing(); + + return true; + }); + } private void FileRenamed(StringView oldFilePath, StringView newFilePath) @@ -546,7 +572,6 @@ class AssetHierarchy if (oldFilePath.EndsWith(AssetFile.ConfigFileExtension)) return; - String oldFileNameWithContentRoot = scope .(); Path.InternalCombine(oldFileNameWithContentRoot, _assetsDirectory, oldFilePath); @@ -560,7 +585,10 @@ class AssetHierarchy if (File.Exists(oldConfigFileName) && !File.Exists(newConfigFileName)) { - if (File.Move(oldConfigFileName, newConfigFileName) case .Err(let value)) + // TODO: Actually Move, but it's a bit complicated to manage cases, where external programs move the original file and + // rename an new file to the original name. So just copy the config to retain it. + // I don't think there is a clean way to solve this. + if (File.Copy(oldConfigFileName, newConfigFileName) case .Err(let value)) { Log.EngineLogger.Error($"Failed to move file {oldConfigFileName} to {newConfigFileName}. Code: {value}"); } diff --git a/GlitchyEditor/src/Assets/Importers/TextureImporter.bf b/GlitchyEditor/src/Assets/Importers/TextureImporter.bf index fcbfbc5..9973585 100644 --- a/GlitchyEditor/src/Assets/Importers/TextureImporter.bf +++ b/GlitchyEditor/src/Assets/Importers/TextureImporter.bf @@ -222,6 +222,7 @@ class TextureImporter : IAssetImporter } } +[BonTarget] enum GenerateMipMaps { No, @@ -717,7 +718,10 @@ class TextureLoader : IProcessedAssetLoader Runtime.NotImplemented(); } - result.SamplerState = SamplerStateManager.GetSampler(sampler); + using (SamplerState samplerState = SamplerStateManager.GetSampler(sampler)) + { + result.SamplerState = samplerState; + } return result; } diff --git a/GlitchyEditor/src/Assets/TextureAssetLoader.bf b/GlitchyEditor/src/Assets/TextureAssetLoader.bf index 2b44122..196dcef 100644 --- a/GlitchyEditor/src/Assets/TextureAssetLoader.bf +++ b/GlitchyEditor/src/Assets/TextureAssetLoader.bf @@ -26,6 +26,8 @@ class TextureAssetPropertiesEditor : AssetPropertiesEditor public override void ShowEditor() { + return; + if (_textureConfig == null) return; diff --git a/GlitchyEditor/src/EditWindows/PropertiesWindow.bf b/GlitchyEditor/src/EditWindows/PropertiesWindow.bf index 5777cdc..7259fe2 100644 --- a/GlitchyEditor/src/EditWindows/PropertiesWindow.bf +++ b/GlitchyEditor/src/EditWindows/PropertiesWindow.bf @@ -102,6 +102,8 @@ class PropertiesWindow : EditorWindow private void ShowPropertiesEditor(AssetFile assetFile) { + return; + if (_currentPropertiesEditor == null) return; diff --git a/GlitchyEditor/src/EditorContentManager.bf b/GlitchyEditor/src/EditorContentManager.bf index b47db90..0c13a22 100644 --- a/GlitchyEditor/src/EditorContentManager.bf +++ b/GlitchyEditor/src/EditorContentManager.bf @@ -26,7 +26,7 @@ class EditorContentManager : IContentManager private append Dictionary _handleToAsset = .(); private append AssetHierarchy _assetHierarchy = .(this); - private append AssetCache _assetCache = .() ~ delete:append _; + private append AssetCache _assetCache = .(this); private append AssetConverter _assetConverter = .(this); private append List _reloadQueue = .(); @@ -54,9 +54,18 @@ class EditorContentManager : IContentManager if (assetNode.AssetFile?.LoadedAsset == null) return; - _reloadQueue.Add(assetNode.AssetFile.LoadedAsset.Handle); + QueueAssetReload(assetNode.AssetFile.LoadedAsset.Handle); } - + + public void QueueAssetReload(AssetHandle assetHandle) + { + // The asset isn't loaded, so we don't need to reload. + if (!_handleToAsset.ContainsKey(assetHandle)) + return; + + _reloadQueue.Add(assetHandle); + } + public void OnFileRenamed(AssetNode assetNode, StringView oldIdentifier) { // Asset isn't loaded so we don't need to reload it. @@ -373,33 +382,45 @@ class EditorContentManager : IContentManager Log.EngineLogger.AssertDebug(oldAsset != null); - StringView oldIdentifier = oldAsset.Identifier; - - GetResourceAndSubassetName(oldIdentifier, let resourceName, let subassetName); - Result> resultNode = AssetHierarchy.GetNodeFromAssetHandle(handle); if (resultNode case .Err) { - Log.EngineLogger.Error($"Could not find asset node for asset \"{oldIdentifier}\"."); + Log.EngineLogger.Error($"Could not find asset node for asset \"{oldAsset.Identifier}\"."); return; } AssetNode assetNode = resultNode->Value; AssetFile file = assetNode.AssetFile; + + CachedAsset cacheEntry = _assetCache.GetCacheEntry(handle); + + Asset loadedAsset; + + // TODO: Remove this check once we no longer need the old stuff + if (cacheEntry != null) + { + loadedAsset = LoadFromCache(handle, false); + } + else + { + IAssetLoader assetLoader = GetAssetLoader(file); + + Stream stream = OpenStream(assetNode.Path, true); + + StringView oldIdentifier = oldAsset.Identifier; + + GetResourceAndSubassetName(oldIdentifier, let resourceName, let subassetName); + + // TODO: Add async loading! + loadedAsset = assetLoader.LoadAsset(stream, file.AssetConfig.Config, resourceName, subassetName, this); - IAssetLoader assetLoader = GetAssetLoader(file); - - Stream stream = OpenStream(assetNode.Path, true); - - // TODO: Add async loading! - Asset loadedAsset = assetLoader.LoadAsset(stream, file.AssetConfig.Config, resourceName, subassetName, this); - - delete stream; + delete stream; + } if (loadedAsset == null) return; - + file.[Friend]_loadedAsset = loadedAsset; SwapAsset(oldAsset, loadedAsset); @@ -910,7 +931,7 @@ class EditorContentManager : IContentManager } // TODO: obviously use a map or something... - TextureLoader textureLoader = new .(); + TextureLoader textureLoader = new .() ~ delete _; private IProcessedAssetLoader GetLoader(AssetType assetType) { diff --git a/GlitchyEngine/src/Application.bf b/GlitchyEngine/src/Application.bf index b393836..c3b4439 100644 --- a/GlitchyEngine/src/Application.bf +++ b/GlitchyEngine/src/Application.bf @@ -30,7 +30,7 @@ namespace GlitchyEngine private IContentManager _contentManager; - private append List _jobQueue = .() ~ ClearAndDeleteItems!(_); + private append List _jobQueue = .() ~ ClearAndDeleteItems!(_); private append Monitor _jobQueueMutex = .(); public bool IsRunning => _running; @@ -233,7 +233,7 @@ namespace GlitchyEngine } /// Executes the given Job on the main thread. Takes ownership of the delegate. - public void InvokeOnMainThread(delegate void() ownJob) + public void InvokeOnMainThread(delegate bool() ownJob) { using (_jobQueueMutex.Enter()) { @@ -249,10 +249,14 @@ namespace GlitchyEngine { for (let job in _jobQueue) { - job(); + if (job()) + { + @job.RemoveFast(); + delete job; + } } - ClearAndDeleteItems!(_jobQueue); + //ClearAndDeleteItems!(_jobQueue); } } } diff --git a/GlitchyEngine/src/Content/AssetCompression.bf b/GlitchyEngine/src/Content/AssetCompression.bf index 28a39d8..a4b7ce7 100644 --- a/GlitchyEngine/src/Content/AssetCompression.bf +++ b/GlitchyEngine/src/Content/AssetCompression.bf @@ -1,7 +1,10 @@ +using Bon; + namespace GlitchyEngine.Content; +[BonTarget] enum AssetCompression : uint8 { None, - L4Z + LZ4 } diff --git a/GlitchyEngine/src/Scripting/ScriptEngine.bf b/GlitchyEngine/src/Scripting/ScriptEngine.bf index f6f8b3e..52bdfa9 100644 --- a/GlitchyEngine/src/Scripting/ScriptEngine.bf +++ b/GlitchyEngine/src/Scripting/ScriptEngine.bf @@ -257,6 +257,8 @@ static class ScriptEngine _requestingReload = false; _userAssemblyWatcher.StartRaisingEvents(); + + return true; }); });