Basic benchmarking

GlitchyEngine.Benchmark/BeefProj.toml

@@ -0,0 +1,9 @@
+FileVersion = 1
+Dependencies = {corlib = "*", "Beef.Linq" = "*", GlitchyEngine = "*"}
+
+[Project]
+Name = "GlitchyEngine.Benchmark"
+StartupObject = "GlitchyEngine.Benchmark.BenchmarkApp"
+
+[Configs.Debug.Win64]
+DebugWorkingDirectory = "D:\\Development\\Projects\\Beef\\GlitchyEngine\\GlitchyEditor"

GlitchyEngine.Benchmark/src/BasicBenchmark.bf

@@ -0,0 +1,155 @@
+using GlitchyEngine.World;
+using GlitchyEngine.Scripting;
+using System;
+using System.Collections;
+using System.IO;
+using System.Diagnostics;
+namespace GlitchyEngine.Benchmark;
+
+class BasicBenchmark
+{
+	public static void Setup()
+	{
+		Console.WriteLine("Starting up ScriptEngine...");
+		Console.WriteLine(Directory.GetCurrentDirectory(.. scope .()));
+		ScriptEngine.Init();
+		
+		ScriptEngine.SetAppAssemblyPath(@"..\GlitchyEngine.Test\content\TestProject\.cache\bin\TestProject.dll");
+	}
+
+	protected static void Shutdown()
+	{
+		Console.WriteLine("Shutting down ScriptEngine...");
+		ScriptEngine.Shutdown();
+	}
+	
+	/*public static void BenchmarkSerializeEmptyScene()
+	{
+		Setup();
+
+		Scene scene = new Scene();
+		defer scene.ReleaseRef();
+
+		ScriptInstanceSerializer serializer = scope .();
+
+		scene.Start(true, true);
+
+		Stopwatch time = scope Stopwatch(startNow: true);
+
+		serializer.SerializeScriptInstances();
+		
+		time.Stop();
+
+		Console.WriteLine($"Serialized in {time.ElapsedMicroseconds}μs ({time.ElapsedMilliseconds}ms)");
+
+		scene.Stop();
+		
+		Shutdown();
+	}*/
+
+	public static void BenchmarkSerializer()
+	{
+		// Before benchmark
+		Setup();
+
+		BenchmarkResult resultCollector = scope .();
+
+		Scene scene = new Scene();
+		defer scene.ReleaseRef();
+
+		ScriptInstanceSerializer serializer = null;
+
+		let benchmark = scope Benchmark();
+		benchmark.BeforeAll = scope [&]() => {
+			scene.Start(true, true);
+		};
+		benchmark.BeforeRun = scope [&]() => {
+			serializer = new .();
+		};
+		benchmark.Run = scope [&]() => {
+			serializer.SerializeScriptInstances();
+		};
+		benchmark.AfterRun = scope [&]() => {
+			delete serializer;
+		};
+		benchmark.AfterAll = scope [&]() => {
+			scene.Stop();
+		};
+		
+		Console.WriteLine("Benchmark empty scene:");
+		{
+			benchmark.Name = "Empty Scene";
+			benchmark.Run(resultCollector);
+		}
+
+		List<Entity> references = new List<Entity>(1000);
+		defer delete references;
+
+		void BenchmarkTrivialEntity(int entityCount)
+		{
+			Console.WriteLine($"Benchmark {entityCount} Trivial Entity:");
+
+			for (int i < entityCount)
+			{
+				readonly Entity entity = scene.CreateEntity("Test");
+				let script = entity.AddComponent<ScriptComponent>();
+				script.ScriptClassName = "TestProject.TestScript";
+
+				references.Add(entity);
+			}
+			
+			benchmark.Name = scope $"Trivial Entity {entityCount}";
+			benchmark.Run(resultCollector);
+
+			for (Entity e in references)
+			{
+				scene.DestroyEntity(e);
+			}
+
+			references.Clear();
+		}
+
+		BenchmarkTrivialEntity(1);
+		BenchmarkTrivialEntity(10);
+		BenchmarkTrivialEntity(100);
+		BenchmarkTrivialEntity(1000);
+
+		void BenchmarkLargeEntity(int entityCount)
+		{
+			Console.WriteLine($"Benchmark {entityCount} Large Entity:");
+
+			for (int i < entityCount)
+			{
+				readonly Entity entity = scene.CreateEntity("Test");
+				let script = entity.AddComponent<ScriptComponent>();
+				script.ScriptClassName = "TestProject.SerializationTest";
+				
+				references.Add(entity);
+			}
+			
+			benchmark.Name = scope $"Large Entity {entityCount}";
+			benchmark.Run(resultCollector);
+
+			for (Entity e in references)
+			{
+				scene.DestroyEntity(e);
+			}
+
+			references.Clear();
+		}
+
+		BenchmarkLargeEntity(1);
+		BenchmarkLargeEntity(10);
+		BenchmarkLargeEntity(100);
+		BenchmarkLargeEntity(1000);
+
+		// After Bench
+		Shutdown();
+		
+		String output = scope .();
+		resultCollector.PrintStats(output);
+		File.WriteAllText("test.log", output);
+
+		Console.WriteLine(output);
+	}
+}

GlitchyEngine.Benchmark/src/Benchmark.bf

@@ -0,0 +1,228 @@
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Collections;
+using System.Collections;
+
+namespace GlitchyEngine.Benchmark;
+
+class BenchmarkResult
+{
+	public class BenchmarkRun
+	{
+		public String Name  { get; private set; } ~ delete:append _;
+		public TimeSpan[] RunDurations  { get; private set; } ~ delete:append _;
+		
+		public int RunCount => RunDurations.Count;
+
+		public TimeSpan TotalRunTime { get; private set; }
+		public TimeSpan MeanRunTime { get; private set; }
+		public TimeSpan RuntimeStdDev { get; private set; }
+
+		[AllowAppend]
+		public this(StringView name, int runCount)
+		{
+			String appendedName = append String(name);
+			TimeSpan[] appendedRunDurations = append TimeSpan[runCount];
+
+			Name = appendedName;
+			RunDurations = appendedRunDurations;
+		}
+
+		public void CalculateStats()
+		{
+			TotalRunTime = RunDurations.Sum();
+			MeanRunTime = TimeSpan(TotalRunTime.Ticks / RunCount);
+
+			int64 variance = RunDurations.Select(scope (d) => {
+					TimeSpan delta = d - MeanRunTime;
+					return (delta * delta).Ticks / RunCount;
+				}).Sum();
+			RuntimeStdDev = TimeSpan((int64)Math.Sqrt((double)variance));
+		}
+	}
+
+	private append List<BenchmarkRun> _runData ~ ClearAndDeleteItems!(_);
+
+	public this()
+	{
+	}
+
+	public BenchmarkRun NewRun(StringView name, int iterations)
+	{
+		BenchmarkRun newRunResult = new BenchmarkRun(name, iterations);
+		_runData.Add(newRunResult);
+
+		return newRunResult;
+	}
+
+	private enum TimeUnit
+	{
+		Milliseconds,
+		Seconds,
+		Minutes,
+		Hours,
+		Days
+	}
+
+	private TimeUnit FigureOutUnits<T>(T timeData) where T : concrete, IEnumerable<TimeSpan>
+	{
+		TimeSpan shortestMeanTime = _runData.Select(scope (s) => s.MeanRunTime).Min();
+
+		if (shortestMeanTime.TotalDays > 2)
+		{
+			return .Days;
+		}
+		else if (shortestMeanTime.TotalHours > 2)
+		{
+			return .Hours;
+		}
+		else if (shortestMeanTime.TotalMinutes > 5)
+		{
+			return .Minutes;
+		}
+		else if (shortestMeanTime.TotalSeconds > 10)
+		{
+			return .Seconds;
+		}
+		else
+		{
+			return .Milliseconds;
+		}
+	}
+
+	private void PrintTimeSpan(TimeSpan timeToPrint, TimeUnit unitToPrint, String outBuffer)
+	{
+		switch (unitToPrint)
+		{
+		case .Milliseconds:
+			outBuffer.AppendF($"{timeToPrint.TotalMilliseconds:f3} ms");
+		case .Seconds:
+			outBuffer.AppendF($"{timeToPrint.TotalSeconds:f3} s");
+		case .Minutes:
+			outBuffer.AppendF($"{timeToPrint.TotalMinutes:f3} m");
+		case .Hours:
+			outBuffer.AppendF($"{timeToPrint.TotalHours:f3} h");
+		case .Days:
+			outBuffer.AppendF($"{timeToPrint.TotalDays:f3} d");
+		}
+	}
+
+	public void PrintStats(String outString, bool printHeader = true, TimeUnit? forcedTimeUnit = null)
+	{
+		for (BenchmarkRun run in _runData)
+		{
+			run.CalculateStats();
+		}
+
+		TimeUnit printedUnit = forcedTimeUnit ?? FigureOutUnits(_runData.Select(scope (s) => s.MeanRunTime));
+
+		var widths = (NameColumn: 8, TotalTimeColumn: 7, MeanTimeColumn: 7, StdDevColumn: 7);
+
+		SimpleStringList preparedStrings = scope .();
+		for (BenchmarkRun run in _runData)
+		{
+			// Print Name
+			widths.NameColumn = Math.Max(widths.NameColumn, run.Name.Length);
+
+			// Print Total Time
+			StringView totalRunTimeView = preparedStrings.Add(scope (s) => PrintTimeSpan(run.TotalRunTime, printedUnit, s));
+			widths.TotalTimeColumn = Math.Max(widths.TotalTimeColumn, totalRunTimeView.Length);
+
+			// Print Total Time
+			StringView meanRunTimeView = preparedStrings.Add(scope (s) => PrintTimeSpan(run.MeanRunTime, printedUnit, s));
+			widths.MeanTimeColumn = Math.Max(widths.MeanTimeColumn, meanRunTimeView.Length);
+
+			// Print Standard Deviation
+			StringView stdDevView = preparedStrings.Add(scope (s) => PrintTimeSpan(run.RuntimeStdDev, printedUnit, s));
+			widths.StdDevColumn = Math.Max(widths.StdDevColumn, stdDevView.Length);
+		}
+
+		String lineFormat = scope $"| {{, -{widths.NameColumn}}} | {{, {widths.TotalTimeColumn}}} | {{, {widths.MeanTimeColumn}}} | {{, {widths.StdDevColumn}}} |\n";
+
+		if (printHeader)
+		{
+			int lengthBefore = outString.Length;
+			outString.AppendF(lineFormat, Formatter.Center("Benchmark", widths.NameColumn), Formatter.Center("Total", widths.TotalTimeColumn),
+				Formatter.Center("Mean", widths.MeanTimeColumn), Formatter.Center("Std", widths.StdDevColumn));
+			int lineLength = outString.Length - lengthBefore;
+			outString.Append('-', lineLength - 1);
+			outString.Append('\n');
+		}
+		
+		int preparedStringIndex = 0;
+		for (BenchmarkRun run in _runData)
+		{
+			outString.AppendF(lineFormat, run.Name, preparedStrings[preparedStringIndex++], preparedStrings[preparedStringIndex++], preparedStrings[preparedStringIndex++]);
+		}
+	}
+}
+
+class Benchmark
+{
+	private append String _name = .();
+
+	public StringView Name
+	{
+		get => _name;
+		set => _name.Set(value);
+	}
+
+	public Action BeforeAll;
+	public Action BeforeRun;
+	public Action Run;
+	public Action AfterRun;
+	public Action AfterAll;
+
+	public void Run(BenchmarkResult resultsCollector, int warmupRuns = 10, int runs = 100)
+	{
+		let resultData = resultsCollector.NewRun(Name, runs);
+
+		BeforeAll?.Invoke();
+
+		Stopwatch watch = scope .();
+
+		watch.Start();
+		for (int i < warmupRuns)
+		{
+			Console.Write($"Warmup run {i + 1}/{warmupRuns}...");
+			Console.CursorLeft = 0;
+			//TimeSpan duration = DoRun();
+			//Console.WriteLine($"Warmup run {i + 1} finished after {duration.TotalMilliseconds}ms.");
+		}
+		watch.Stop();
+		Console.WriteLine($"Warmup finished after {watch.Elapsed.TotalMilliseconds}ms");
+
+		for (int i < runs)
+		{
+			Console.Write($"Starting run {i + 1}/{runs}...");
+			Console.CursorLeft = 0;
+
+			watch.Restart();
+			resultData.RunDurations[i] = DoRun();
+			watch.Stop();
+
+			//Console.WriteLine($"Run {i + 1}: {_runDurations[i]} (total: {watch.Elapsed.TotalMilliseconds}ms).");
+		}
+
+		resultData.CalculateStats();
+
+		Console.WriteLine(scope $"Finished: Total ({runs} runs) {resultData.TotalRunTime.TotalMilliseconds}ms | Mean: {resultData.MeanRunTime.TotalMilliseconds}ms | Std Dev: {resultData.RuntimeStdDev.TotalMilliseconds}ms");
+
+		AfterAll?.Invoke();
+	}
+
+	private TimeSpan DoRun()
+	{
+		Stopwatch watch = scope .();
+		BeforeRun?.Invoke();
+
+		watch.Start();
+		Run.Invoke();
+		watch.Stop();
+
+		AfterRun?.Invoke();
+
+		return watch.Elapsed;
+	}
+}

GlitchyEngine.Benchmark/src/BenchmarkApp.bf

@@ -0,0 +1,18 @@
+using System;
+
+namespace GlitchyEngine.Benchmark;
+
+class BenchmarkApp
+{
+	public static void Main(String[] args)
+	{
+		BasicBenchmark.BenchmarkSerializer();
+	}
+
+	/// Provide a stub implementation for CreateApplication, so that the linker is happy.
+	[Export, LinkName("CreateApplication")]
+	public static Application CreateApplication(String[] args)
+	{
+		return null;
+	}
+}

GlitchyEngine.Benchmark/src/Formatter.bf

@@ -0,0 +1,47 @@
+using System;
+
+namespace GlitchyEngine.Benchmark;
+
+static class Formatter
+{
+	public static CenterFormatter<T> Center<T>(T value, int width, bool leftAlignWhenUneven = true)
+	{
+		return CenterFormatter<T>(value, width, leftAlignWhenUneven);
+	}
+	
+	public struct CenterFormatter<T> : IFormattable
+	{
+		private T _value;
+		private int _width;
+		private bool _leftAlignWhenUneven;
+
+		public this(T value, int width, bool leftAlignWhenUneven = true)
+		{
+			_value = value;
+			_width = width;
+			_leftAlignWhenUneven = leftAlignWhenUneven;
+		}
+
+		public void ToString(String outString, String format, IFormatProvider formatProvider)
+		{
+			String buffer = scope String(128);
+			IFormattable formattableArg = _value as IFormattable;
+			if (formattableArg != null)
+				formattableArg.ToString(buffer, format, formatProvider);
+			else if (_value != null)
+				_value.ToString(buffer);
+			else
+				buffer.Append("null");
+
+			int missingSpaceCount = _width - buffer.Length;
+
+			int widthAfterLeftPad = _width - missingSpaceCount / 2;
+			// If the number of spaces is uneven and we want to left align (have more space right) we need to pad one less on the left
+			if (_leftAlignWhenUneven && missingSpaceCount % 2 == 1)
+				widthAfterLeftPad -= 1;
+		 
+			buffer..PadLeft(widthAfterLeftPad);
+			outString..Append(buffer).PadRight(_width, ' ');
+		}
+	}
+}

GlitchyEngine.Benchmark/src/SimpleStringList.bf

@@ -0,0 +1,54 @@
+using System;
+using System.Collections;
+
+namespace GlitchyEngine.Benchmark;
+
+class SimpleStringList
+{
+	private append String _buffer = .();
+	private append List<(int start, int length)> _views = .();
+
+	public StringView this[int index]
+	{
+		get
+		{
+			let (start, length) = _views[index];
+			return StringView(_buffer, start, length);
+		}
+	}
+
+	public StringView Add(StringView text)
+	{
+		int lengthBefore = _buffer.Length;
+
+		_buffer.Append(text);
+		
+		StringView newStringView = StringView(_buffer, lengthBefore);
+		_views.Add((lengthBefore, newStringView.Length));
+		
+		// Append a null terminator to ensure that every string can easily be a C string.
+		// Append after creating string view so it isn't part of the view.
+		_buffer.Append('\0');
+
+		return newStringView;
+	}
+
+	public delegate void CustomToString(String buffer);
+
+	// Allows adding a string directly into the buffer, using the internal buffer as target for e.g. ToString-opeations.
+	public StringView Add(CustomToString toString)
+	{
+		int lengthBefore = _buffer.Length;
+
+		toString(_buffer);
+
+		StringView newStringView = StringView(_buffer, lengthBefore);
+		_views.Add((lengthBefore, newStringView.Length));
+
+		// Append a null terminator to ensure that every string can easily be a C string.
+		// Append after creating string view so it isn't part of the view.
+		_buffer.Append('\0');
+
+		return newStringView;
+	}
+}