advanced async copy of files

This commit is contained in:
Simon Lübeß
2025-06-26 10:16:39 +02:00
parent bfd1754d14
commit 108e24eb38
11 changed files with 1202 additions and 28 deletions
@@ -0,0 +1,76 @@
using System;
using System.IO;
using System.Threading;
using System.Collections;
using System.Diagnostics;
using ImGui;
namespace GlitchyEditor.Multithreading;
using internal GlitchyEditor.Multithreading;
abstract class BackgroundTask
{
internal BackgroundTaskList _list;
internal BackgroundTask _previous;
internal BackgroundTask _next;
internal BackgroundTaskManager _taskManager;
public enum RunState
{
Ready,
Blocked,
Paused,
Running,
Aborted,
Finished
}
public enum RunResult
{
/// The task gives up it's runtime, but has to run again.
Continue,
/// The task gives up it's runtime, because it cannot continue without user intervention through the UI.
Pause,
/// The task finished and wont run again.
Finished,
/// The operation was aborted, the task will not run again.
Abort
}
public RunState State { get; internal set; } = .Ready;
public bool Ready => State == .Ready;
public bool Paused => State == .Paused;
public bool Blocked => State == .Blocked;
public bool Running => State == .Running;
public bool Aborted => State == .Aborted;
public bool Finished => State == .Finished;
public bool Ended => Finished || Aborted;
public bool DeleteWhenStopped { get; set; }
public abstract RunResult Run();
public abstract void OnRenderPopup();
public void Continue()
{
_taskManager.ContinueTask(this);
}
public void Pause()
{
if (State == .Ready || State == .Running)
{
State = .Paused;
}
}
public void Abort()
{
_taskManager.AbortTask(this);
}
}
@@ -0,0 +1,422 @@
using System;
using System.Diagnostics;
namespace GlitchyEditor.Multithreading;
using internal GlitchyEditor.Multithreading;
class BackgroundTaskList
{
BackgroundTask _sentinel = new BackgroundTask()
{
public override BackgroundTask.RunResult Run() => .Finished;
public override void OnRenderPopup() { }
_next = _,
_previous = _,
_list = this
} ~ delete _;
private int _count;
public BackgroundTask First => _sentinel._next == _sentinel ? null : _sentinel._next;
public BackgroundTask Last => _sentinel._previous == _sentinel ? null : _sentinel._previous;
public BackgroundTask End => _sentinel;
public int Count => _count;
private void InsertBefore(BackgroundTask item, BackgroundTask beforeNode)
{
item._next = beforeNode;
item._previous = beforeNode._previous;
item._next._previous = item;
item._previous._next = item;
item._list = this;
_count++;
Validate();
}
public void AddFirst(BackgroundTask item)
{
if (item._list != null)
item._list.Remove(item, false);
InsertBefore(item, _sentinel._next);
}
public void AddLast(BackgroundTask item)
{
if (item._list != null)
item._list.Remove(item, false);
InsertBefore(item, _sentinel);
}
public void Clear()
{
Clear(false);
}
public void Clear(bool deleteElements)
{
BackgroundTask task = _sentinel._next;
while (task != _sentinel)
{
BackgroundTask nextTask = task._next;
UnlinkElement(task);
if (deleteElements)
delete task;
task = nextTask;
}
_sentinel._next = _sentinel._previous = _sentinel;
_count = 0;
Validate();
}
private void UnlinkElement(BackgroundTask element)
{
element._next = null;
element._previous = null;
element._list = null;
}
public bool Contains(BackgroundTask item)
{
return item._list == this;
}
public bool Remove(BackgroundTask item)
{
return Remove(item, false);
}
public bool Remove(BackgroundTask item, bool deleteElement)
{
if (item._list != this)
return false;
item._previous._next = item._next;
item._next._previous = item._previous;
UnlinkElement(item);
if (deleteElement)
delete item;
_count--;
Validate();
return true;
}
public Result<BackgroundTask> TryPopFront()
{
if (_sentinel._next == _sentinel)
return .Err;
BackgroundTask first = _sentinel._next;
Remove(first, false);
return first;
}
public Result<BackgroundTask> TryPopBack()
{
if (_sentinel._previous == _sentinel)
return .Err;
BackgroundTask first = _sentinel._previous;
Remove(first, false);
return first;
}
internal void Validate()
{
BackgroundTask task = _sentinel._next;
int visitedNodes = 0;
while (task != _sentinel)
{
visitedNodes++;
Debug.Assert(visitedNodes <= Count);
Debug.Assert(task._list == this);
BackgroundTask nextTask = task._next;
Debug.Assert(nextTask._previous == task);
task = nextTask;
}
Debug.Assert(visitedNodes == Count);
}
}
class BackgroundTaskListTests
{
// Stub implementation
class BackgroundTask : GlitchyEditor.Multithreading.BackgroundTask
{
public override BackgroundTask.RunResult Run()
{
return default;
}
public override void OnRenderPopup()
{
}
}
static void TestBasicOperations()
{
Debug.WriteLine("🧪 Testing Basic Operations...");
var list1 = scope BackgroundTaskList();
var task1 = scope BackgroundTask();
var task2 = scope BackgroundTask();
// Test AddFirst
list1.AddFirst(task1);
Debug.Assert(list1.Count == 1);
Debug.Assert(task1._list == list1);
Debug.Assert(list1.Contains(task1));
list1.Validate();
// Test AddLast
list1.AddLast(task2);
Debug.Assert(list1.Count == 2);
Debug.Assert(task2._list == list1);
Debug.Assert(list1.Contains(task2));
list1.Validate();
// Test Remove
list1.Remove(task1);
Debug.Assert(list1.Count == 1);
Debug.Assert(task1._list == null);
Debug.Assert(!list1.Contains(task1));
Debug.Assert(list1.Contains(task2));
list1.Validate();
Debug.WriteLine("✅ Basic Operations passed!");
}
static void TestAutoRemoveFromOldList()
{
Debug.WriteLine("🧪 Testing Auto-Remove from Old List...");
var list1 = scope BackgroundTaskList();
var list2 = scope BackgroundTaskList();
var task = scope BackgroundTask();
// Add to list1
list1.AddFirst(task);
Debug.Assert(list1.Count == 1);
Debug.Assert(list2.Count == 0);
Debug.Assert(task._list == list1);
list1.Validate();
list2.Validate();
// Add to list2 - should remove from list1
list2.AddLast(task);
Debug.Assert(list1.Count == 0);
Debug.Assert(list2.Count == 1);
Debug.Assert(task._list == list2);
Debug.Assert(!list1.Contains(task));
Debug.Assert(list2.Contains(task));
list1.Validate();
list2.Validate();
// add to list1 again, should be removed from list2
list1.AddFirst(task);
Debug.Assert(list1.Count == 1);
Debug.Assert(list2.Count == 0);
Debug.Assert(task._list == list1);
list1.Validate();
list2.Validate();
Debug.WriteLine("✅ Auto-Remove passed!");
}
static void TestTryPopFront()
{
Debug.WriteLine("🧪 Testing TryPopFront...");
var list = scope BackgroundTaskList();
// Test empty list
let emptyResult = list.TryPopFront();
Debug.Assert(emptyResult == .Err);
list.Validate();
var task1 = scope BackgroundTask();
var task2 = scope BackgroundTask();
var task3 = scope BackgroundTask();
// Add three elements
list.AddLast(task1);
list.Validate();
list.AddLast(task2);
list.Validate();
list.AddLast(task3);
list.Validate();
Debug.Assert(list.Count == 3);
// Pop first element
let pop1 = list.TryPopFront();
Debug.Assert(pop1 == .Ok(task1));
Debug.Assert(list.Count == 2);
Debug.Assert(task1._list == null);
list.Validate();
// Pop second element
let pop2 = list.TryPopFront();
Debug.Assert(pop2 == .Ok(task2));
Debug.Assert(list.Count == 1);
Debug.Assert(task2._list == null);
list.Validate();
// Pop third element
let pop3 = list.TryPopFront();
Debug.Assert(pop3 == .Ok(task3));
Debug.Assert(list.Count == 0);
Debug.Assert(task3._list == null);
list.Validate();
// Pop on now empty list
let pop4 = list.TryPopFront();
Debug.Assert(pop4 == .Err);
list.Validate();
Debug.WriteLine("✅ TryPopFront passed!");
}
/*static void TestIterator()
{
Debug.WriteLine("🧪 Testing Iterator...");
var list = scope BackgroundTaskList();
var tasks = scope BackgroundTask*[5];
for (int i = 0; i < tasks.Count; i++)
{
tasks[i] = new BackgroundTask();
list.AddLast(tasks[i]);
}
defer { for (var t in tasks) delete t; }
// Teste Iterator
int count = 0;
for (var task in list)
{
Debug.Assert(task == tasks[count]);
count++;
}
Debug.Assert(count == 5);
Debug.WriteLine("✅ Iterator passed!");
}*/
static void TestClear()
{
Debug.WriteLine("🧪 Testing Clear...");
var list = scope BackgroundTaskList();
var task1 = scope BackgroundTask();
var task2 = scope BackgroundTask();
var task3 = scope BackgroundTask();
list.AddLast(task1);
list.Validate();
list.AddLast(task2);
list.Validate();
list.AddLast(task3);
list.Validate();
Debug.Assert(list.Count == 3);
// Clear sollte task1 und task3 löschen, aber nicht task2
list.Clear();
Debug.Assert(list.Count == 0);
list.Validate();
Debug.WriteLine("✅ Clear passed!");
}
static void TestComplexScenario()
{
// Simulate processing queues.
Debug.WriteLine("🧪 Testing Complex Scenario...");
var pending = scope BackgroundTaskList();
var processing = scope BackgroundTaskList();
var completed = scope BackgroundTaskList();
// Add to pending
var tasks = scope BackgroundTask[10];
for (int i = 0; i < tasks.Count; i++)
{
tasks[i] = scope:: BackgroundTask();
pending.AddLast(tasks[i]);
pending.Validate();
}
// Move from pending to processing
while (true)
{
let result = pending.TryPopFront();
if (result == .Err) break;
let task = result.Value;
processing.AddLast(task);
processing.Validate();
}
Debug.Assert(pending.Count == 0);
Debug.Assert(processing.Count == 10);
// Move to completed
while (true)
{
let result = processing.TryPopFront();
if (result == .Err) break;
completed.AddLast(result.Value);
completed.Validate();
}
Debug.Assert(processing.Count == 0);
Debug.Assert(completed.Count == 10);
Debug.WriteLine("✅ Complex Scenario passed!");
}
public static void RunTests()
{
Debug.WriteLine("🚀 Starting BackgroundTaskList Tests...\n");
TestBasicOperations();
TestAutoRemoveFromOldList();
TestTryPopFront();
//TestIterator();
TestClear();
TestComplexScenario();
Debug.WriteLine("\n🎉 All tests passed!");
}
}
@@ -0,0 +1,216 @@
using System;
using System.Threading;
using System.Collections;
using GlitchyEngine;
using GlitchyEngine.Collections;
namespace GlitchyEditor.Multithreading;
using internal GlitchyEditor.Multithreading;
// This would go nicely with some light weight threading!
class BackgroundTaskManager
{
private Thread _workerThread;
private bool _run;
private BackgroundTask _runningTask;
private append BackgroundTaskList _readyQueue = .();
private append BackgroundTaskList _waitingTasks = .();
private append Monitor _queueLock = .();
public void Init()
{
_run = true;
_workerThread = new Thread(new => WorkerLoop);
_workerThread.Start();
}
public void Deinit()
{
_run = false;
// TODO: Delete jobs if necessary
// TODO: Danger if thread is deadlocked.
_workerThread.Join();
}
private void WorkerLoop()
{
while (_run)
{
Result<BackgroundTask> nextRunningTask = .Err;
using (_queueLock.Enter())
{
nextRunningTask = _readyQueue.TryPopFront();
}
if (nextRunningTask case .Ok(let task))
{
_runningTask = task;
_runningTask.State = .Running;
BackgroundTask.RunResult runResult = task.Run();
_runningTask = null;
switch (runResult)
{
case .Continue:
task.State = .Ready;
using (_queueLock.Enter())
{
_readyQueue.AddLast(task);
}
case .Pause:
task.State = .Paused;
using (_queueLock.Enter())
{
_waitingTasks.AddLast(task);
}
case .Finished:
task.State = .Finished;
if (task.DeleteWhenStopped)
{
delete task;
}
case .Abort:
task.State = .Aborted;
if (task.DeleteWhenStopped)
{
delete task;
}
}
}
else
{
Thread.Sleep(10);
}
}
}
public Result<void> StartBackgroundTask(BackgroundTask task)
{
if (task._taskManager != null)
{
Log.EngineLogger.Error("Task is already managed by a taskmanager.");
return .Err;
}
if (task.State == .Running || task.State == .Aborted)
{
Log.EngineLogger.Error("Task State illegal. (Did it already run?)");
return .Err;
}
task._taskManager = this;
using (_queueLock.Enter())
{
if (task.State == .Ready)
_readyQueue.AddLast(task);
else
_waitingTasks.AddLast(task);
}
return .Ok;
}
internal void ContinueTask(BackgroundTask task)
{
Log.EngineLogger.AssertDebug(task._taskManager == this);
Log.EngineLogger.AssertDebug(!task.Ended);
Application.Instance.InvokeOnMainThread(new () =>
{
if (task.State == .Paused)
{
task.State = .Ready;
using (_queueLock.Enter())
{
Log.EngineLogger.AssertDebug(!_readyQueue.Contains(task));
// We assume this was a user interaction so this task will get some priority.
_readyQueue.AddFirst(task);
}
}
return true;
});
}
internal void Pause(BackgroundTask task)
{
Log.EngineLogger.AssertDebug(task._taskManager == this);
Log.EngineLogger.AssertDebug(!task.Ended);
Application.Instance.InvokeOnMainThread(new () =>
{
if (task.State == .Ready)
{
task.State = .Ready;
using (_queueLock.Enter())
{
Log.EngineLogger.AssertDebug(!_waitingTasks.Contains(task));
_waitingTasks.AddLast(task);
}
}
return true;
});
}
internal void AbortTask(BackgroundTask task)
{
Log.EngineLogger.AssertDebug(task._taskManager == this);
Log.EngineLogger.AssertDebug(!task.Ended);
Application.Instance.InvokeOnMainThread(new () =>
{
if (task.State != .Aborted)
{
task.State = .Aborted;
using (_queueLock.Enter())
{
_waitingTasks.Remove(task);
}
if (task.DeleteWhenStopped)
{
delete task;
}
}
return true;
});
}
public void ImGuiRender()
{
using (_queueLock.Enter())
{
_runningTask?.OnRenderPopup();
BackgroundTask task = _readyQueue.First;
while (task != null && task != _readyQueue.End)
{
Log.EngineLogger.Assert(task._list == _readyQueue);
task.OnRenderPopup();
task = task._next;
}
BackgroundTask wtask = _waitingTasks.First;
while (wtask != null && wtask != _waitingTasks.End)
{
Log.EngineLogger.Assert(wtask._list == _waitingTasks);
wtask.OnRenderPopup();
wtask = wtask._next;
}
}
}
}
@@ -0,0 +1,320 @@
using System;
using System.IO;
using System.Collections;
using System.Diagnostics;
using ImGui;
namespace GlitchyEditor.Multithreading;
class CopyBackgroundTask : BackgroundTask
{
private List<String> _sourcePath ~ {ClearAndDeleteItems!(_); delete:append _;};
private String _targetDirectoryPath ~ delete:append _;
private int _totalEntriesToCopy;
private append Queue<CopyInfo> _pathsToCopy = .() ~ ClearAndDeleteItems!(_);
private append Queue<String> _scanQueue = .() ~ ClearAndDeleteItems!(_);
private enum OverwriteMode
{
None,
OverwriteFile = 1,
KeepBoth = 2,
CombineDirectories = 4,
Skip = 8
}
private OverwriteMode _nextFileMode;
private OverwriteMode _allFilesMode;
private OverwriteMode CurrentFileMode => _nextFileMode | _allFilesMode;
private class CopyInfo
{
public String SourcePath ~ delete:append _;
public String TargetPath ~ delete:append _;
public OverwriteMode OverwriteMode;
[AllowAppend]
public this(StringView sourcePath, StringView targetPath, OverwriteMode overwriteMode)
{
String src = append String(sourcePath);
String tgt = append String(targetPath);
SourcePath = src;
TargetPath = tgt;
OverwriteMode = overwriteMode;
}
}
public bool ScanningFiles => !_scanQueue.IsEmpty;
[AllowAppend]
public this(List<String> sourcePaths, StringView targetDirectoryPath)
{
List<String> sourcePathList = append List<String>(sourcePaths.Count);
String targetDirectoryPathStr = append String(targetDirectoryPath);
_sourcePath = sourcePathList;
_targetDirectoryPath = targetDirectoryPathStr;
for (String path in sourcePaths)
{
_sourcePath.Add(new String(path));
_scanQueue.Add(new String(path));
}
_totalEntriesToCopy = sourcePaths.Count;
}
public enum CopyError : IDisposable
{
case None;
case TargetDirectoryIsNotDirectory;
case SourceDoesntExist(String SourcePath);
case TargetFileExists(String FileName);
case TargetDirectoryExists(String DirectoryName);
public void Dispose()
{
switch (this)
{
case .None:
case .TargetDirectoryIsNotDirectory:
case .SourceDoesntExist(let SourcePath):
delete SourcePath;
case .TargetFileExists(let FileName):
delete FileName;
case .TargetDirectoryExists(let DirectoryName):
delete DirectoryName;
}
}
}
private Result<void, CopyError> CollectFilesToCopy()
{
void RemoveFront()
{
String path = _scanQueue.PopFront();
delete path;
}
while (!_scanQueue.IsEmpty && Running)
{
//Thread.Sleep(1000);
String currentPath = _scanQueue.Peek();
//FileInfo sourceInfo = scope FileInfo(currentPath);
String fileName = scope .();
Path.GetFileName(currentPath, fileName);
String targetPath = scope .();
Path.Combine(targetPath, _targetDirectoryPath, fileName);
//FileInfo targetInfo = scope FileInfo(targetPath);
if (Directory.Exists(currentPath))
{
/*for (FileFindEntry e in Directory.Enumerate(currentPath))
{
String entryPath = new String();
e.GetFilePath(entryPath);
_scanQueue.Add(entryPath);
}*/
}
else if (File.Exists(currentPath))
{
if (File.Exists(targetPath))
{
switch (CurrentFileMode)
{
case .KeepBoth, .OverwriteFile, .Skip:
default:
return .Err(.TargetFileExists(new String(fileName)));
}
}
_pathsToCopy.Add(new CopyInfo(currentPath, targetPath, CurrentFileMode));
RemoveFront();
_nextFileMode = .None;
}
}
_totalEntriesToCopy = _pathsToCopy.Count;
return .Ok;
}
private CopyError _currentError = .None ~ _.Dispose();
public override RunResult Run()
{
_currentError.Dispose();
_currentError = .None;
if (CollectFilesToCopy() case .Err(out _currentError))
{
return .Pause;
}
while (!_pathsToCopy.IsEmpty && Running)
{
//Thread.Sleep(1000);
CopyInfo currentPath = _pathsToCopy.PopFront();
defer
{
delete currentPath;
}
CopyPath(currentPath);
}
return .Finished;
}
private Result<void> CopyPath(CopyInfo sourcePath)
{
if (Directory.Exists(sourcePath.SourcePath))
{
}
else if (File.Exists(sourcePath.SourcePath))
{
bool forceOverwrite = false;
if (File.Exists(sourcePath.TargetPath))
{
switch (sourcePath.OverwriteMode)
{
case .KeepBoth:
String fileExtension = scope .();
Path.GetExtension(sourcePath.TargetPath, fileExtension);
String fileName = scope .();
Path.GetFileName(sourcePath.SourcePath, fileName);
Path.FindFreePath(_targetDirectoryPath, fileName.Substring(0..<^fileExtension.Length), fileExtension, sourcePath.TargetPath..Clear());
case .OverwriteFile:
forceOverwrite = true;
case .Skip:
default:
return .Err;//(.TargetFileExists);
}
}
File.Copy(sourcePath.SourcePath, sourcePath.TargetPath, forceOverwrite);
}
else
{
// TODO error
return .Err;
}
return .Ok;
}
public override void OnRenderPopup()
{
if (ImGui.Begin("Copying files..."))
{
if (ScanningFiles)
{
ImGui.ProgressBar(-1.0f * (float)ImGui.GetTime(), .(-1, 0), scope $"Found {_pathsToCopy.Count} files...");
}
else
{
int copiedFiles = _totalEntriesToCopy - _pathsToCopy.Count;
ImGui.ProgressBar(copiedFiles / _totalEntriesToCopy, .(-1, 0), scope $"Copied {copiedFiles} / {_totalEntriesToCopy} files.");
}
switch (_currentError)
{
case .None:
// Do nothing
case .TargetDirectoryIsNotDirectory:
// This shouldn't be possible
Debug.Break();
case .SourceDoesntExist(let SourcePath):
// This isn't good!
case .TargetFileExists(let fileName):
ImGui.PushStyleColor(.Text, 0xFF0000FF);
ImGui.TextUnformatted(scope $"A file with the name \"{fileName}\" already exists.");
ImGui.PopStyleColor();
if (ImGui.Button("Overwrite"))
{
_nextFileMode = .OverwriteFile;
Continue();
}
ImGui.AttachTooltip("Overwrites the existing file. Will ask again if another conflict occurs.");
if (ImGui.Button("Overwrite All"))
{
_allFilesMode = .OverwriteFile;
Continue();
}
ImGui.AttachTooltip("Overwrites all existing files.");
if (ImGui.Button("Keep both"))
{
_nextFileMode = .KeepBoth;
Continue();
}
ImGui.AttachTooltip("Keeps both the existing file as well as the new one, differentiates by adding a number to the name. Will ask again if another conflict occurs.");
if (ImGui.Button("Keep all"))
{
_allFilesMode = .KeepBoth;
Continue();
}
ImGui.AttachTooltip("Keeps both the existing files as well as the new ones, differentiates by adding a number to the name.");
if (ImGui.Button("Cancel"))
{
Abort();
}
case .TargetDirectoryExists(let directoryName):
ImGui.PushStyleColor(.Text, 0xFF0000FF);
ImGui.TextUnformatted(scope $"A file with the name \"{directoryName}\" already exists.");
ImGui.PopStyleColor();
if (ImGui.Button("Combine"))
{
_nextFileMode = .CombineDirectories;
Continue();
}
ImGui.AttachTooltip("Merges the copied directory into the target directory. Will ask again if another conflict occurs.");
if (ImGui.Button("Combine All"))
{
_allFilesMode = .CombineDirectories;
Continue();
}
ImGui.AttachTooltip("Merges all copied directories into the target directories.");
if (ImGui.Button("Keep both"))
{
_nextFileMode = .KeepBoth;
Continue();
}
ImGui.AttachTooltip("Renames the copied directory, so that they no longer conflict. Will ask again if another conflict occurs.");
if (ImGui.Button("Keep all"))
{
_allFilesMode = .KeepBoth;
Continue();
}
ImGui.AttachTooltip("Renames the copied directory, so that they no longer conflict.");
if (ImGui.Button("Cancel"))
{
Abort();
}
}
ImGui.End();
}
}
}