From 853148e9e7717dbdb29b6dcb5bee510a35fa04d6 Mon Sep 17 00:00:00 2001 From: Enderlook Date: Sat, 20 Nov 2021 19:05:22 -0300 Subject: [PATCH] Initial code --- Net Pools.sln | 25 + Net Pools/Net Pools.csproj | 24 + Net Pools/src/DynamicObjectPool.cs | 445 +++++++++++ Net Pools/src/NetStandard2.0.cs | 7 + Net Pools/src/ObjectPool.cs | 47 ++ Net Pools/src/ObjectPoolHelper.cs | 54 ++ Net Pools/src/ObjectWrapper.cs | 7 + ...dLocalOverPerCoreLockedStacksObjectPool.cs | 688 ++++++++++++++++++ Net Pools/src/Utilities.cs | 120 +++ 9 files changed, 1417 insertions(+) create mode 100644 Net Pools.sln create mode 100644 Net Pools/Net Pools.csproj create mode 100644 Net Pools/src/DynamicObjectPool.cs create mode 100644 Net Pools/src/NetStandard2.0.cs create mode 100644 Net Pools/src/ObjectPool.cs create mode 100644 Net Pools/src/ObjectPoolHelper.cs create mode 100644 Net Pools/src/ObjectWrapper.cs create mode 100644 Net Pools/src/ThreadLocalOverPerCoreLockedStacksObjectPool.cs create mode 100644 Net Pools/src/Utilities.cs diff --git a/Net Pools.sln b/Net Pools.sln new file mode 100644 index 0000000..c9b3e42 --- /dev/null +++ b/Net Pools.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.31829.152 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Net Pools", "Net Pools\Net Pools.csproj", "{009D3D36-FB90-4318-B3FE-85BA53527D31}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {009D3D36-FB90-4318-B3FE-85BA53527D31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {009D3D36-FB90-4318-B3FE-85BA53527D31}.Debug|Any CPU.Build.0 = Debug|Any CPU + {009D3D36-FB90-4318-B3FE-85BA53527D31}.Release|Any CPU.ActiveCfg = Release|Any CPU + {009D3D36-FB90-4318-B3FE-85BA53527D31}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {9D6637E5-BE56-4BE9-A652-6095FCF8FACB} + EndGlobalSection +EndGlobal diff --git a/Net Pools/Net Pools.csproj b/Net Pools/Net Pools.csproj new file mode 100644 index 0000000..1a98459 --- /dev/null +++ b/Net Pools/Net Pools.csproj @@ -0,0 +1,24 @@ + + + + netstandard2.0;netstandard2.1;net5;net6 + Library + Enderlook.Pools + Enderlook.Pools + Enderlook.Pools + Enderlook + Enderlook.Pools + https://github.com/Enderlook/Net-Pools + git + 0.1.0 + 10 + true + true + enable + + + + + + + diff --git a/Net Pools/src/DynamicObjectPool.cs b/Net Pools/src/DynamicObjectPool.cs new file mode 100644 index 0000000..bae88b0 --- /dev/null +++ b/Net Pools/src/DynamicObjectPool.cs @@ -0,0 +1,445 @@ +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Enderlook.Pools +{ + /// + /// A lightweight, fast and thread-safe object pool to store objects.
+ /// The pool is desinged for fast rent and return of element, so during multithreading scenarios it may accidentally free unnecessary objects during return (however, this is not a fatal error). + /// + /// Type of object to pool + public sealed class DynamicObjectPool : ObjectPool where T : class + { + /// + /// Delegate that instantiates new object. + /// + private readonly Func factory; + + /// + /// Storage for the pool objects.
+ /// The array is not an stack so the whole array must be traversed to find objects. + /// + private readonly ObjectWrapper[] array; + + /// + /// The first item is stored in a dedicated field because we expect to be able to satisfy most requests from it. + /// + private T? firstElement; + + /// + /// A dynamic-size stack reserve of objects.
+ /// When get fulls, the first half of it is emptied and its element are moved here.
+ /// When gets empty, the first half of it is fulled with elements from this reserve.
+ /// Those operations are done in a batch to reduce the amount of times this requires to be acceded.
+ /// However, those operations only moves the first half of the array to prevent a point where this is executed on each rent or return. + /// + private ObjectWrapper[]? reserve; + + /// + /// Keep tracks of the amount of used slots in . + /// + private int reserveCount; + + /// + /// Keep record of last time was trimmed; + /// + private int reserveMillisecondsTimeStamp; + + /// + /// Keep record of last time was trimmed; + /// + private int arrayMillisecondsTimeStamp; + + /// + /// Creates a pool of objects. + /// + /// Hot capacity of the pool.
+ /// If this capacity is fulled, the pool will expand a cold region.
+ /// The hot capacity should preferably be not greater than * 2. + /// Initial capacity of the cold pool.
+ /// This reserve pool is only acceded when the hot pool gets full or empty since it's slower.
+ /// This pool has a dynamic size so this value represent the initial size of the pool which may enlarge or shrink over time. + /// Delegate used to construct instances of the pooled objects.
+ /// If no delegate is provided, a factory with the default constructor of will be used. + /// /// Throw when is lower than 1. + public DynamicObjectPool(int hotCapacity, int initialColdCapacity, Func? factory) + { + if (hotCapacity < 1) Utilities.ThrowArgumentOutOfRangeException_HotCapacityCanNotBeLowerThanOne(); + if (initialColdCapacity < 0) Utilities.ThrowArgumentOutOfRangeException_InitialColdCapacityCanNotBeNegative(); + + this.factory = factory ?? ObjectPoolHelper.Factory; + array = new ObjectWrapper[hotCapacity - 1]; // -1 due to firstElement. + reserve = new ObjectWrapper[initialColdCapacity]; + } + + /// + /// Creates a pool of objects. + /// + /// Hot capacity of the pool.
+ /// If this capacity is fulled, the pool will expand a cold region.
+ /// The hot capacity should preferably be not greater than * 2. + /// Delegate used to construct instances of the pooled objects.
+ /// If no delegate is provided, a factory with the default constructor for will be used. + /// Throw when is lower than 1. + public DynamicObjectPool(int hotCapacity, Func? factory) : this(hotCapacity, hotCapacity, factory) { } + + /// + /// Creates a pool of objects. + /// + /// Hot capacity of the pool. If this capacity is fulled, the pool will expand a cold region. + /// Throw when is lower than 1. + public DynamicObjectPool(int hotCapacity) : this(hotCapacity, hotCapacity, null) { } + + /// + /// Creates a pool of objects. + /// + /// Delegate used to construct instances of the pooled objects.
+ /// If no delegate is provided, a factory with the default constructor for will be used. + public DynamicObjectPool(Func? factory) : this(Environment.ProcessorCount * 2, Environment.ProcessorCount * 2, factory) { } + + /// + /// Creates a pool of objects. + /// + public DynamicObjectPool() : this(Environment.ProcessorCount * 2, Environment.ProcessorCount * 2, null) { } + + /// + public override int ApproximateCount() + { + int count = firstElement is null ? 0 : 1; + ObjectWrapper[] items = array; + for (int i = 0; i < items.Length; i++) + if (items[i].Value is not null) + count++; + return count + reserveCount; + } + + /// + public override T Rent() + { + // First, we examine the first element. + // If that fails, we look at the remaining elements. + // Note that intitial read are optimistically not syncronized. This is intentional. + // We will interlock only when we have a candidate. + // In a worst case we may miss some recently returned objects. + T? element = firstElement; + if (element is null || element != Interlocked.CompareExchange(ref firstElement, null, element)) + { + // Next, we look at all remaining elements. + ObjectWrapper[] items = array; + + for (int i = 0; i < items.Length; i++) + { + // Note that intitial read are optimistically not syncronized. This is intentional. + // We will interlock only when we have a candidate. + // In a worst case we may miss some recently returned objects. + element = items[i].Value; + if (element is not null && element == Interlocked.CompareExchange(ref items[i].Value, null, element)) + break; + } + + // Next, we look at the reserve if it has elements. + element = reserveCount > 0 ? FillFromReserve() : factory(); + } + + return element; + } + + /// + /// Return rented object to pool.
+ /// If the pool is full, the object will be discarded. + /// + /// Object to return. + public override void Return(T obj) + { + // Intentionally not using interlocked here. + // In a worst case scenario two objects may be stored into same slot. + // It is very unlikely to happen and will only mean that one of the objects will get collected. + if (firstElement is null) + firstElement = obj; + else + { + ObjectWrapper[] items = array; + for (int i = 0; i < items.Length; i++) + { + if (items[i].Value is null) + { + // Intentionally not using interlocked here. + // In a worst case scenario two objects may be stored into same slot. + // It is very unlikely to happen and will only mean that one of the objects will get collected. + items[i].Value = obj; + return; + } + } + + SendToReserve(obj); + } + } + + /// + public override void Trim(bool force = false) + { + const int ArrayLowAfterMilliseconds = 60 * 1000; // Trim after 60 seconds for low pressure. + const int ArrayMediumAfterMilliseconds = 60 * 1000; // Trim after 60 seconds for medium pressure. + const int ArrayHighTrimAfterMilliseconds = 10 * 1000; // Trim after 10 seconds for high pressure. + const int ArrayLowTrimCount = 1; // Trim one item when pressure is low. + const int ArrayMediumTrimCount = 2; // Trim two items when pressure is moderate. + + const int ReserveLowTrimAfterMilliseconds = 90 * 1000; // Trim after 90 seconds for low pressure. + const int ReserveMediumTrimAfterMilliseconds = 45 * 1000; // Trim after 45 seconds for low pressure. + const float ReserveLowTrimPercentage = .10f; // Trim 10% of objects for low pressure; + const float ReserveMediumTrimPercentage = .30f; // Trim 30% of objects for moderate pressure; + + int currentMilliseconds = Environment.TickCount; + + firstElement = null; // We always trim the first element. + + ObjectWrapper[] items = array; + int length = items.Length; + + int arrayTrimMilliseconds; + int arrayTrimCount; + int reserveTrimMilliseconds; + float reserveTrimPercentage; + if (force) + { + // Forces to clear everything regardless of time. + arrayTrimCount = length; + arrayTrimMilliseconds = 0; + reserveTrimMilliseconds = 0; + reserveTrimPercentage = 1; + } + else + { + switch (Utilities.GetMemoryPressure()) + { + case Utilities.MemoryPressure.High: + arrayTrimCount = length; + arrayTrimMilliseconds = ArrayHighTrimAfterMilliseconds; + // Forces to clear everything regardless of time. + reserveTrimMilliseconds = 0; + reserveTrimPercentage = 1; + break; + case Utilities.MemoryPressure.Medium: + arrayTrimCount = ArrayMediumTrimCount; + arrayTrimMilliseconds = ArrayMediumAfterMilliseconds; + reserveTrimMilliseconds = ReserveMediumTrimAfterMilliseconds; + reserveTrimPercentage = ReserveMediumTrimPercentage; + break; + default: + arrayTrimCount = ArrayLowTrimCount; + arrayTrimMilliseconds = ArrayLowAfterMilliseconds; + reserveTrimMilliseconds = ReserveLowTrimAfterMilliseconds; + reserveTrimPercentage = ReserveLowTrimPercentage; + break; + } + } + + if (arrayMillisecondsTimeStamp == 0) + arrayMillisecondsTimeStamp = currentMilliseconds; + + if ((currentMilliseconds - arrayMillisecondsTimeStamp) > arrayTrimMilliseconds) + { + // We've elapsed enough time since the last clean. + // Drop the top items so they can be collected and make the pool look a little newer. + + if (arrayTrimCount != length) + { + for (int i = 0; i < length; i++) + { + if (items[i].Value is not null) + { + // Intentionally not using interlocked here. + items[i].Value = null; + if (--arrayTrimCount == 0) + { + arrayMillisecondsTimeStamp += arrayMillisecondsTimeStamp / 4; // Give the remaining items a bit more time. + break; + } + } + } + arrayMillisecondsTimeStamp = 0; + } + else + { + firstElement = null; +#if NET6_0_OR_GREATER + Array.Clear(items); +#else + Array.Clear(items, 0, length); +#endif + arrayMillisecondsTimeStamp = 0; + } + } + + if (reserveCount == 0) + reserveMillisecondsTimeStamp = 0; + else + { + int reserveCount_; + // Under high pressure, we don't wait time to trim, so we remove all objects in reserve. + if (reserveTrimPercentage == 1) + { + Debug.Assert(reserveTrimMilliseconds == 0); + ObjectWrapper[]? reserve_; + do + { + reserve_ = Interlocked.Exchange(ref reserve, null); + } while (reserve_ is null); + reserveCount_ = 0; + + if (reserve_.Length <= items.Length) + { +#if NET6_0_OR_GREATER + Array.Clear(reserve_); +#else + Array.Clear(reserve_, 0, reserve_.Length); +#endif + } + else + reserve_ = new ObjectWrapper[reserve_.Length]; + + reserveMillisecondsTimeStamp = 0; + reserveCount = reserveCount_; + reserve = reserve_; + } + else + { + if (reserveMillisecondsTimeStamp == 0) + reserveMillisecondsTimeStamp = currentMilliseconds; + + if ((currentMilliseconds - reserveMillisecondsTimeStamp) > reserveTrimMilliseconds) + { + // Otherwise, remove a percentage of all stored objects in the reserve, based on how long was the last trimming. + // This time is approximate, with the time set not when the element is stored but when we see it during a Trim, + // so it takes at least two Trim calls (and thus two gen2 GCs) to drop elements, unless we're in high memory pressure. + + ObjectWrapper[]? reserve_; + do + { + reserve_ = Interlocked.Exchange(ref reserve, null); + } while (reserve_ is null); + reserveCount_ = reserveCount; + + int toRemove = (int)Math.Ceiling(reserveCount_ * reserveTrimPercentage); + int newReserveCount = Math.Max(reserveCount_ - toRemove, 0); + toRemove = reserveCount_ - newReserveCount; + int reserveLength = reserve_.Length; + reserveCount_ = newReserveCount; + + // Since the global reserve has a dynamic size, we shrink the reserve if it gets too small. + if (reserveLength / reserveCount_ >= 4) + { + if (reserveLength <= items.Length) + goto simpleClean; + else + { + int newLength = Math.Min(reserveLength / 2, items.Length); + ObjectWrapper[] array = new ObjectWrapper[newLength]; + Array.Copy(reserve_, array, newReserveCount); + reserve_ = array; + goto next2; + } + } + simpleClean: + Array.Clear(reserve_, newReserveCount, toRemove); + next2:; + + reserveCount = reserveCount_; + reserve = reserve_; + } + } + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private T FillFromReserve() + { + ObjectWrapper[] items = array; + ObjectWrapper[]? reserve_; + do + { + reserve_ = Interlocked.Exchange(ref reserve, null); + } while (reserve_ is null); + + int oldCount = reserveCount; + int count = oldCount; + if (count > 0) + { + T? element = reserve_[--count].Value; + Debug.Assert(element is not null); + for (int i = 0; i < items.Length / 2 && count > 0; i++) + { + // Note that intitial read and write are optimistically not syncronized. This is intentional. + // We will interlock only when we have a candidate. + // In a worst case we may miss some recently returned objects or accidentally free objects. + if (items[i].Value is null) + items[i].Value = reserve_[--count].Value; + } + + Array.Clear(reserve_, count, oldCount - count); + + reserveCount = count; + reserve = reserve_; + return element; + } + else + { + reserve = reserve_; + return factory(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void SendToReserve(T obj) + { + ObjectWrapper[] items = array; + ObjectWrapper[]? reserve_; + do + { + reserve_ = Interlocked.Exchange(ref reserve, null); + } while (reserve_ is null); + + int count = reserveCount; + if (count + 1 + (items.Length / 2) > reserve_.Length) + Array.Resize(ref reserve_, Math.Max(reserve_.Length * 2, 1)); + + reserve_[count++].Value = obj; + + for (int i = 0; i < items.Length / 2 && count < reserve_.Length; i++) + { + // We don't use an optimistically not syncronized initial read in this part. + // This is because we expect the majority of the array to be filled. + // So it's not worth doing an initial read to check that. + T? element = Interlocked.Exchange(ref items[i].Value, null); + if (element is not null) + reserve_[count++].Value = element; + } + + reserveCount = count; + reserve = reserve_; + } + + private sealed class GCCallback + { + private readonly GCHandle owner; + + public GCCallback(DynamicObjectPool owner) => this.owner = GCHandle.Alloc(owner, GCHandleType.Weak); + + ~GCCallback() + { + object? owner = this.owner.Target; + if (owner is null) + this.owner.Free(); + else + { + Debug.Assert(owner is DynamicObjectPool); + Unsafe.As>(owner).Trim(); + GC.ReRegisterForFinalize(this); + } + } + } + } +} diff --git a/Net Pools/src/NetStandard2.0.cs b/Net Pools/src/NetStandard2.0.cs new file mode 100644 index 0000000..f22bded --- /dev/null +++ b/Net Pools/src/NetStandard2.0.cs @@ -0,0 +1,7 @@ +#if !(NETSTANDARD2_1_OR_GREATER || NET5_0_OR_GREATER) +namespace System.Diagnostics.CodeAnalysis +{ + [AttributeUsage(AttributeTargets.Method, Inherited = false)] + internal sealed class DoesNotReturnAttribute : Attribute { } +} +#endif \ No newline at end of file diff --git a/Net Pools/src/ObjectPool.cs b/Net Pools/src/ObjectPool.cs new file mode 100644 index 0000000..7155ac1 --- /dev/null +++ b/Net Pools/src/ObjectPool.cs @@ -0,0 +1,47 @@ +using System.Runtime.CompilerServices; + +namespace Enderlook.Pools +{ + /// + /// Represent a pool of objects. + /// + /// Type of object to pool + public abstract class ObjectPool where T : class + { + /// + /// Retrieves a shared instance. + /// + public static ObjectPool Shared { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => ThreadLocalOverPerCoreLockedStacksObjectPool.Singlenton; + } + + /// + /// Gets an approximate count of the objects stored in the pool.
+ /// This value is not accurate and may be lower or higher than the actual count.
+ /// This is primary used for debugging purposes. + /// + /// Approximate count of elements in the pool. + public abstract int ApproximateCount(); + + /// + /// Rent an element from the pool.
+ /// If the pool is empty, instantiate a new element. + /// + /// Rented element. + public abstract T Rent(); + + /// + /// Return an element to the pool.
+ /// If the pool is full, it's an implementation detail whenever the object is free or the pool is resized.
+ /// If is , it's an implementation detail whenever it throws an exception or ignores. + /// + public abstract void Return(T obj); + + /// + /// Trim the content of the pool. + /// + /// If , the pool is forced to clear all elements inside. Otherwise, the pool may only clear partially or not clear at all if the heuristic says so. + public abstract void Trim(bool force = false); + } +} diff --git a/Net Pools/src/ObjectPoolHelper.cs b/Net Pools/src/ObjectPoolHelper.cs new file mode 100644 index 0000000..dfaf3dc --- /dev/null +++ b/Net Pools/src/ObjectPoolHelper.cs @@ -0,0 +1,54 @@ +using System; +using System.Linq.Expressions; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; + +namespace Enderlook.Pools +{ + internal static class ObjectPoolHelper where T : class + { + public static readonly Func Factory; + + static ObjectPoolHelper() + { + // TODO: In .NET 7 Activator.CreateFactory() may be added https://github.com/dotnet/runtime/issues/36194. + + ConstructorInfo? constructor = typeof(T).GetConstructor(Type.EmptyTypes); + if (constructor is null) + { + Factory = () => throw new MissingMethodException($"No parameterless constructor defined for type '{typeof(T)}'."); + return; + } + + switch (Utilities.DynamicCompilationMode) + { + case Utilities.SystemLinqExpressions: + Factory = Expression.Lambda>(Expression.New(typeof(T)), Array.Empty()).Compile(); + break; + +#if NETSTANDARD2_1_OR_GREATER || NET5_0_OR_GREATER + case Utilities.SystemReflectionEmitDynamicMethod: + + DynamicMethod dynamicMethod = new DynamicMethod("Instantiate", typeof(T), Type.EmptyTypes); + ILGenerator generator = dynamicMethod.GetILGenerator(); + generator.Emit(OpCodes.Newobj, constructor); + generator.Emit(OpCodes.Ret); +#if NET5_0_OR_GREATER + Factory = dynamicMethod.CreateDelegate>(); +#else + Factory = (Func)dynamicMethod.CreateDelegate(typeof(Func)); +#endif + break; +#endif + default: + Factory = () => Activator.CreateInstance(); + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Create() + => Utilities.DynamicCompilationMode == Utilities.DisabledDynamicCompilation ? Activator.CreateInstance() : Factory(); + } +} diff --git a/Net Pools/src/ObjectWrapper.cs b/Net Pools/src/ObjectWrapper.cs new file mode 100644 index 0000000..514d3fd --- /dev/null +++ b/Net Pools/src/ObjectWrapper.cs @@ -0,0 +1,7 @@ +namespace Enderlook.Pools +{ + internal struct ObjectWrapper // Prevent runtime covariant checks on array access. + { + public T Value; + } +} diff --git a/Net Pools/src/ThreadLocalOverPerCoreLockedStacksObjectPool.cs b/Net Pools/src/ThreadLocalOverPerCoreLockedStacksObjectPool.cs new file mode 100644 index 0000000..1ba8c84 --- /dev/null +++ b/Net Pools/src/ThreadLocalOverPerCoreLockedStacksObjectPool.cs @@ -0,0 +1,688 @@ +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Enderlook.Pools +{ + /// + /// A fast and thread-safe object pool to store a large amount of objects. + /// + /// Type of object to pool + internal sealed class ThreadLocalOverPerCoreLockedStacksObjectPool : ObjectPool where T : class + { + // Inspired from https://source.dot.net/#System.Private.CoreLib/TlsOverPerCoreLockedStacksArrayPool.cs + + /// + /// Maximum length of to use. + /// + private const int MaximumPerCoreStack = 64; // Selected to avoid needing to worry about processor groups. + + /// + /// The maximum number of objects to store in each per-core stack. + /// + private const int MaxObjectsPerCore = 128; + + /// + /// The initial capacity of . + /// + private const int InitialGlobalReserveCapacity = 256; + + /// + /// Number of locked stacks to employ. + /// + private static readonly int PerCoreStacksCount = Math.Min(Environment.ProcessorCount, MaximumPerCoreStack); + + /// + /// A per-thread element for better cache. + /// + [ThreadStatic] + private static ThreadLocalElement? threadLocalElement; + + /// + /// Used to keep tack of all thread local objects for trimming if needed. + /// + private static GCHandle[]? allThreadLocalElements = new GCHandle[Environment.ProcessorCount]; + + /// + /// Keep tracks of the amount of used slots in . + /// + private static int allThreadLocalElementsCount; + + /// + /// An array of per-core objects.
+ /// The slots are lazily initialized. + /// + private static readonly PerCoreStack[] perCoreStacks = new PerCoreStack[PerCoreStacksCount]; + + /// + /// A global dynamic-size reserve of elements.
+ /// When all get fulls, one of them is emptied and all its objects are moved here.
+ /// When all get empty, one of them is fulled with objects from this reserve.
+ /// Those operations are done in a batch to reduce the amount of times this requires to be acceded. + /// + private static ObjectWrapper[]? globalReserve = new ObjectWrapper[MaxObjectsPerCore]; + + /// + /// Keep tracks of the amount of used slots in . + /// + private static int globalReserveCount; + + /// + /// Keep record of last time was trimmed; + /// + private static int globalReserveMillisecondsTimeStamp; + + /// + /// Unique instance of this type. + /// + // Store the shared ObjectPool in a field of its derived sealed type so the Jit can "see" the exact type + // when the Shared property on ObjectPool is inlined which will allow it to devirtualize calls made on it. + public static readonly ThreadLocalOverPerCoreLockedStacksObjectPool Singlenton = new(); + + private ThreadLocalOverPerCoreLockedStacksObjectPool() { } + + static ThreadLocalOverPerCoreLockedStacksObjectPool() + { + for (int i = 0; i < perCoreStacks.Length; i++) + perCoreStacks[i] = new PerCoreStack(new ObjectWrapper[MaxObjectsPerCore]); + GCCallback _ = new(); + } + + /// + public override int ApproximateCount() + { + int count = 0; + for (int i = 0; i < perCoreStacks.Length; i++) + count += perCoreStacks[i].GetCount(); + + ObjectWrapper[]? globalReserve_; + do + { + globalReserve_ = Interlocked.Exchange(ref globalReserve, null); + } while (globalReserve_ is null); + count += globalReserveCount; + globalReserve = globalReserve_; + + return count; + } + + /// + public override T Rent() + { + // First, try to get an element from the thread local field if possible. + ThreadLocalElement? threadLocalElement_ = threadLocalElement; + if (threadLocalElement_ is not null) + { + T? element = threadLocalElement_.Value; + if (element is not null) + { + threadLocalElement_.Value = null; + return element; + } + } + + // Next, try to get an element from one of the per-core stacks. + PerCoreStack[] perCoreStacks_ = perCoreStacks; + // Try to pop from the associated stack first. + // If that fails, try with other stacks. +#if NETSTANDARD2_1_OR_GREATER || NET5_0_OR_GREATER + int currentProcessorId = Thread.GetCurrentProcessorId(); +#else + int currentProcessorId = Thread.CurrentThread.ManagedThreadId; // TODO: This is probably a bad idea. +#endif + int index = (int)((uint)currentProcessorId % (uint)PerCoreStacksCount); + for (int i = 0; i < perCoreStacks_.Length; i++) + { + T? element = perCoreStacks_[index].TryPop(); + if (element is not null) + return element; + + if (++index == perCoreStacks_.Length) + index = 0; + } + + // Next, try to fill a per-core stack with objects from the global reserve. + if (globalReserveCount > 0) + return FillFromGlobalReserve(); + + // Finally, instantiate a new object. + return ObjectPoolHelper.Create(); + + [MethodImpl(MethodImplOptions.NoInlining)] + T FillFromGlobalReserve() + { + T? element = perCoreStacks_[index].FillFromGlobalReserve(); + if (element is not null) + return element; + // Finally, instantiate a new object. + return ObjectPoolHelper.Create(); + } + } + + /// + /// Return rented object to pool.
+ /// If the pool is full, the object will be discarded. + /// + /// Object to return. + public override void Return(T obj) + { + if (obj is null) Utilities.ThrowArgumentNullException_Obj(); + Debug.Assert(obj is not null); + + // Store the element into the thread local field. + // If there's already an object in it, push that object down into the per-core stacks, + // preferring to keep the latest one in thread local field for better locality. + ThreadLocalElement threadLocalElement_ = threadLocalElement ?? ThreadLocalOverPerCoreLockedStacksObjectPool.InitializeThreadLocalElement(); + T? previous = threadLocalElement_.Value; + threadLocalElement_.Value = obj; + threadLocalElement_.MillisecondsTimeStamp = 0; + if (previous is not null) + { + // Try to store the object from one of the per-core stacks. + PerCoreStack[] perCoreStacks_ = perCoreStacks; + // Try to push from the associated stack first. + // If that fails, try with other stacks. +#if NETSTANDARD2_1_OR_GREATER || NET5_0_OR_GREATER + int currentProcessorId = Thread.GetCurrentProcessorId(); +#else + int currentProcessorId = Thread.CurrentThread.ManagedThreadId; // TODO: This is probably a bad idea. +#endif + int index = (int)((uint)currentProcessorId % (uint)PerCoreStacksCount); + for (int i = 0; i < perCoreStacks_.Length; i++) + { + if (perCoreStacks_[index].TryPush(obj)) + return; + + if (++index == perCoreStacks_.Length) + index = 0; + } + + // Next, transfer a per-core stack to the global reserve. + perCoreStacks_[index].MoveToGlobalReserve(obj); + } + } + + /// + public override void Trim(bool force = false) + { + // This method does nothing to prevent user trying to clear the singlenton. + } + + /// + private static void Trim_(bool force = false) + { + const int PerCoreLowTrimAfterMilliseconds = 60 * 1000; // Trim after 60 seconds for low pressure. + const int PerCoreMediumTrimAfterMilliseconds = 60 * 1000; // Trim after 60 seconds for moderate pressure. + const int PerCoreHighTrimAfterMilliseconds = 10 * 1000; // Trim after 10 seconds for high pressure. + const int PerCoreLowTrimCount = 1; // Trim 1 item when pressure is low. + const int PerCoreMediumTrimCount = 2; // Trim 2 items when pressure is moderate. + const int PerCoreHighTrimCount = MaxObjectsPerCore; // Trim all items when pressure is high. + + const int ThreadLocalLowMilliseconds = 30 * 1000; // Trim after 30 seconds for moderate pressure. + const int ThreadLocalMediumMilliseconds = 15 * 1000; // Trim after 15 seconds for low pressure. + + const int ReserveLowTrimAfterMilliseconds = 90 * 1000; // Trim after 90 seconds for low pressure. + const int ReserveMediumTrimAfterMilliseconds = 45 * 1000; // Trim after 45 seconds for low pressure. + const float ReserveLowTrimPercentage = .10f; // Trim 10% of elements for low pressure; + const float ReserveMediumTrimPercentage = .30f; // Trim 30% of elements for moderate pressure; + + int currentMilliseconds = Environment.TickCount; + + Utilities.MemoryPressure memoryPressure; + int perCoreTrimMilliseconds; + int perCoreTrimCount; + uint threadLocalTrimMilliseconds; + int globalTrimMilliseconds; + float globalTrimPercentage; + if (force) + { + memoryPressure = Utilities.MemoryPressure.High; + perCoreTrimCount = PerCoreHighTrimCount; + // Forces to clear everything regardless of time. + perCoreTrimMilliseconds = 0; + threadLocalTrimMilliseconds = 0; + globalTrimMilliseconds = 0; + globalTrimPercentage = 1; + } + else + { + memoryPressure = Utilities.GetMemoryPressure(); + switch (memoryPressure) + { + case Utilities.MemoryPressure.High: + perCoreTrimCount = PerCoreHighTrimCount; + perCoreTrimMilliseconds = PerCoreHighTrimAfterMilliseconds; + // Forces to clear everything regardless of time. + threadLocalTrimMilliseconds = 0; + globalTrimMilliseconds = 0; + globalTrimPercentage = 1; + break; + case Utilities.MemoryPressure.Medium: + perCoreTrimCount = PerCoreMediumTrimCount; + perCoreTrimMilliseconds = PerCoreMediumTrimAfterMilliseconds; + threadLocalTrimMilliseconds = ThreadLocalMediumMilliseconds; + globalTrimMilliseconds = ReserveMediumTrimAfterMilliseconds; + globalTrimPercentage = ReserveMediumTrimPercentage; + break; + default: + Debug.Assert(memoryPressure == Utilities.MemoryPressure.Low); + perCoreTrimCount = PerCoreLowTrimCount; + perCoreTrimMilliseconds = PerCoreLowTrimAfterMilliseconds; + threadLocalTrimMilliseconds = ThreadLocalLowMilliseconds; + globalTrimMilliseconds = ReserveLowTrimAfterMilliseconds; + globalTrimPercentage = ReserveLowTrimPercentage; + break; + } + } + + // Trim each of the per-core stacks. + for (int i = 0; i < perCoreStacks.Length; i++) + perCoreStacks[i].TryTrim(currentMilliseconds, perCoreTrimMilliseconds, perCoreTrimCount); + + // Trim each of the thread local fields. + // Note that threads may be modifying their thread local fields concurrently with this trimming happening. + // We do not force synchronization with those operations, so we accept the fact + // that we may potentially trim an object we didn't need to. + // Both of these should be rare occurrences. + + GCHandle[]? allThreadLocalElements_; + do + { + allThreadLocalElements_ = Interlocked.Exchange(ref allThreadLocalElements, null); + } while (allThreadLocalElements_ is null); + int length = allThreadLocalElementsCount; + int count = 0; + + // Under high pressure, we don't wait time to trim, so we release all thread locals. + if (threadLocalTrimMilliseconds == 0) + { + for (int i = 0; i < length; i++) + { + GCHandle handle = allThreadLocalElements_[i]; + object? target = handle.Target; + if (target is null) + { + handle.Free(); + continue; + } + Debug.Assert(target is ThreadLocalElement); + Unsafe.As(target).Clear(); + allThreadLocalElements_[count++] = handle; + } + } + else + { + // Otherwise, release thread locals based on how long we've observed them to be stored. + // This time is approximate, with the time set not when the object is stored but when we see it during a Trim, + // so it takes at least two Trim calls (and thus two gen2 GCs) to drop an object, unless we're in high memory pressure. + + for (int i = 0; i < length; i++) + { + GCHandle handle = allThreadLocalElements_[i]; + object? target = handle.Target; + if (target is null) + { + handle.Free(); + continue; + } + Debug.Assert(target is ThreadLocalElement); + Unsafe.As(target).Trim(currentMilliseconds, threadLocalTrimMilliseconds); + allThreadLocalElements_[count++] = handle; + } + } + + allThreadLocalElementsCount = count; + allThreadLocalElements = allThreadLocalElements_; + + ObjectWrapper[]? globalReserve_; + do + { + globalReserve_ = Interlocked.Exchange(ref globalReserve, null); + } while (globalReserve_ is null); + int globalCount = globalReserveCount; + + if (globalCount == 0) + globalReserveMillisecondsTimeStamp = 0; + else + { + // Under high pressure, we don't wait time to trim, so we remove all objects in reserve. + if (globalTrimPercentage == 1) + { + Debug.Assert(globalTrimMilliseconds == 0); + globalCount = 0; + if (globalReserve_.Length <= MaxObjectsPerCore) + { +#if NET6_0_OR_GREATER + Array.Clear(globalReserve_); +#else + Array.Clear(globalReserve_, 0, globalReserve_.Length); +#endif + } + else + globalReserve_ = new ObjectWrapper[InitialGlobalReserveCapacity]; + globalReserveMillisecondsTimeStamp = 0; + } + else + { + if (globalReserveMillisecondsTimeStamp == 0) + globalReserveMillisecondsTimeStamp = currentMilliseconds; + + if ((currentMilliseconds - globalReserveMillisecondsTimeStamp) > globalTrimMilliseconds) + { + // Otherwise, remove a percentage of all stored objects in the reserve, based on how long was the last trimming. + // This time is approximate, with the time set not when the object is stored but when we see it during a Trim, + // so it takes at least two Trim calls (and thus two gen2 GCs) to drop objects, unless we're in high memory pressure. + + int toRemove = (int)Math.Ceiling(globalCount * globalTrimPercentage); + int newGlobalCount = Math.Max(globalCount - toRemove, 0); + toRemove = globalCount - newGlobalCount; + int globalLength = globalReserve_.Length; + globalCount = newGlobalCount; + + // Since the global reserve has a dynamic size, we shrink the reserve if it gets too small. + if (globalLength / newGlobalCount >= 4) + { + if (globalLength <= InitialGlobalReserveCapacity) + goto simpleClean; + else + { + int newLength = globalLength / 2; + ObjectWrapper[] array = new ObjectWrapper[newLength]; + Array.Copy(globalReserve_, array, newGlobalCount); + globalReserve_ = array; + goto next; + } + } + simpleClean: + Array.Clear(globalReserve_, newGlobalCount, toRemove); + next:; + } + } + } + + globalReserveCount = globalCount; + globalReserve = globalReserve_; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ThreadLocalElement InitializeThreadLocalElement() + { + ThreadLocalElement slot = new(); + threadLocalElement = slot; + + GCHandle[]? allThreadLocalElements_; + do + { + allThreadLocalElements_ = Interlocked.Exchange(ref allThreadLocalElements, null); + } while (allThreadLocalElements_ is null); + + int count = allThreadLocalElementsCount; + if (unchecked((uint)count >= (uint)allThreadLocalElements_.Length)) + { + count = 0; + for (int i = 0; i < allThreadLocalElements_.Length; i++) + { + GCHandle handle = allThreadLocalElements_[i]; + object? target = handle.Target; + if (target is null) + { + handle.Free(); + continue; + } + allThreadLocalElements_[count++] = handle; + } + + if (count < allThreadLocalElements_.Length) + Array.Clear(allThreadLocalElements_, count, allThreadLocalElements_.Length - count); + else + Array.Resize(ref allThreadLocalElements_, allThreadLocalElements_.Length * 2); + } + + allThreadLocalElements_[count] = GCHandle.Alloc(slot, GCHandleType.Weak); + allThreadLocalElementsCount = count + 1; + + allThreadLocalElements = allThreadLocalElements_; + return slot; + } + + private sealed class ThreadLocalElement + { + public T? Value; + public int MillisecondsTimeStamp; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T? ReplaceWith(T value) + { + T? previous = Value; + Value = value; + MillisecondsTimeStamp = 0; + return previous; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public void Clear() + { + Value = null; + MillisecondsTimeStamp = 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public void Trim(int currentMilliseconds, uint millisecondsThreshold) + { + // We treat 0 to mean it hasn't yet been seen in a Trim call. + // In the very rare case where Trim records 0, it'll take an extra Trim call to remove the object. + int lastSeen = MillisecondsTimeStamp; + if (lastSeen == 0) + MillisecondsTimeStamp = currentMilliseconds; + else if ((currentMilliseconds - lastSeen) >= millisecondsThreshold) + { + // Time noticeably wrapped, or we've surpassed the threshold. + // Clear out the array. + Value = null; + } + } + } + + private struct PerCoreStack + { + private ObjectWrapper[] array; + private int count; + private int millisecondsTimeStamp; + + public PerCoreStack(ObjectWrapper[] array) + { + this.array = array; + count = 0; + millisecondsTimeStamp = 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetCount() + { + int count_; + do + { + count_ = count; + } while (count_ == -1); + return count_; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryPush(T element) + { + ObjectWrapper[] items = array; + + int count_; + do + { + count_ = Interlocked.Exchange(ref count, -1); + } while (count_ == -1); + + bool enqueued = false; + if (unchecked((uint)count_ < (uint)items.Length)) + { + if (count_ == 0) + { + // Reset the time stamp now that we're transitioning from empty to non-empty. + // Trim will see this as 0 and initialize it to the current time when Trim is called. + millisecondsTimeStamp = 0; + } + + items[count_].Value = element; + count_++; + enqueued = true; + } + + count = count_; + return enqueued; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T? TryPop() + { + ObjectWrapper[] items = array; + + int count_; + do + { + count_ = Interlocked.Exchange(ref count, -1); + } while (count_ == -1); + + T? element = null; + int newCount = count_ - 1; + if (unchecked((uint)newCount < (uint)items.Length)) + { + element = items[newCount].Value; + count_ = newCount; + } + + count = count_; + return element; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T? FillFromGlobalReserve() + { + int count_; + do + { + count_ = Interlocked.Exchange(ref count, -1); + } while (count_ == -1); + + ObjectWrapper[]? globalReserve_; + do + { + globalReserve_ = Interlocked.Exchange(ref globalReserve, null); + } while (globalReserve_ is null); + + T? element = null; + int globalCount = globalReserveCount; + if (globalCount > 0) + { + element = globalReserve_[--globalCount].Value; + Debug.Assert(element is not null); + + ObjectWrapper[] items = array; + + int length = Math.Min(MaxObjectsPerCore - count, globalCount); + int start = globalCount - length; + Array.Copy(globalReserve_, start, items, count, length); + Array.Clear(globalReserve_, start, length); + + globalCount = start; + count += length; + + globalReserveCount = globalCount; + } + + globalReserve = globalReserve_; + count = count_; + return element; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void MoveToGlobalReserve(T obj) + { + int count_; + do + { + count_ = Interlocked.Exchange(ref count, -1); + } while (count_ == -1); + + ObjectWrapper[]? globalReserve_; + do + { + globalReserve_ = Interlocked.Exchange(ref globalReserve, null); + } while (globalReserve_ is null); + + ObjectWrapper[] items = array; + int amount = count_ + 1; + int globalCount = globalReserveCount; + int newGlobalCount = globalCount + amount; + if (unchecked((uint)newGlobalCount >= (uint)globalReserve_.Length)) + Array.Resize(ref globalReserve_, globalReserve_.Length * 2); + Array.Copy(items, 0, globalReserve_, globalCount, count_); +#if NET6_0_OR_GREATER + Array.Clear(items); +#else + Array.Clear(items, 0, items.Length); +#endif + globalCount += count_; + count_ = 0; + globalReserve_[globalCount++].Value = obj; + + globalReserveCount = globalCount; + globalReserve = globalReserve_; + count = count_; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void TryTrim(int currentMilliseconds, int trimMilliseconds, int trimCount) + { + if (count == 0) + return; + + ObjectWrapper[] items = array; + + int count_; + do + { + count_ = Interlocked.Exchange(ref count, -1); + } while (count_ == -1); + + if (count_ == 0) + goto end; + + if (millisecondsTimeStamp == 0) + millisecondsTimeStamp = currentMilliseconds; + + if ((currentMilliseconds - millisecondsTimeStamp) <= trimMilliseconds) + goto end; + + // We've elapsed enough time since the first item went into the stack. + // Drop the top item so it can be collected and make the stack look a little newer. + + Array.Clear(items, 0, Math.Min(count_, trimCount)); + count_ = Math.Max(count_ - trimCount, 0); + + millisecondsTimeStamp = count_ > 0 ? + millisecondsTimeStamp + (trimMilliseconds / 4) // Give the remaining items a bit more time. + : 0; + + end: + count = count_; + } + } + + private sealed class GCCallback + { + ~GCCallback() + { + ThreadLocalOverPerCoreLockedStacksObjectPool.Trim_(); + GC.ReRegisterForFinalize(this); + } + } + } +} diff --git a/Net Pools/src/Utilities.cs b/Net Pools/src/Utilities.cs new file mode 100644 index 0000000..88de36a --- /dev/null +++ b/Net Pools/src/Utilities.cs @@ -0,0 +1,120 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; + +namespace Enderlook.Pools +{ + internal static class Utilities + { + public const int DisabledDynamicCompilation = 0; + public const int SystemLinqExpressions = 1; + public const int SystemReflectionEmitDynamicMethod = 2; + public static readonly int DynamicCompilationMode; + + static Utilities() + { + const BindingFlags flags = BindingFlags.Public | BindingFlags.Static; + + // Check if we are in AOT. + // We can't just try to compile code dynamically and try/catch an exception because the runtime explodes instead of throwing in Unity WebGL. + foreach (Assembly assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + if (assembly.FullName == "UnityEngine, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null") + { + Type? applicationType = assembly.GetType("UnityEngine.Application"); + if (applicationType is null) // The trimmer has removed the type, so think about the worst possible case. + goto isDisabled; + + PropertyInfo? platformProperty = applicationType.GetProperty("platform", flags); + if (platformProperty is null) // The trimmer has removed the property, so think about the worst possible case. + goto isDisabled; + + PropertyInfo? isEditorProperty = applicationType.GetProperty("isEditor", flags); + if (isEditorProperty is null) // The trimmer has removed the property, so think about the worst possible case. + goto isDisabled; + + if (platformProperty.GetValue(null)!.ToString() == "WebGLPlayer" && !(bool)isEditorProperty.GetValue(null)!) + goto isDisabled; + } + } + +#if NETSTANDARD2_1_OR_GREATER || NET5_0_OR_GREATER + if (!RuntimeFeature.IsDynamicCodeCompiled) + goto isDisabled; +#endif + + // Try with different approaches beacuse the current platform may not support some of them. + try + { + Expression.Lambda>(Expression.New(typeof(object)), Array.Empty()).Compile()(); + DynamicCompilationMode = SystemLinqExpressions; + return; + } + catch { } + +#if NETSTANDARD2_1_OR_GREATER || NET5_0_OR_GREATER + try + { + DynamicMethod dynamicMethod = new DynamicMethod("Instantiate", typeof(object), Type.EmptyTypes); + ILGenerator generator = dynamicMethod.GetILGenerator(); + generator.Emit(OpCodes.Newobj, typeof(object).GetConstructor(Type.EmptyTypes)!); + generator.Emit(OpCodes.Ret); +#if NET5_0_OR_GREATER + dynamicMethod.CreateDelegate>()(); +#else + ((Func)dynamicMethod.CreateDelegate(typeof(Func)))(); +#endif + DynamicCompilationMode = SystemReflectionEmitDynamicMethod; + return; + } + catch { } +#endif + + isDisabled: + DynamicCompilationMode = DisabledDynamicCompilation; + } + + public enum MemoryPressure + { + Low, + Medium, + High + } + + public static MemoryPressure GetMemoryPressure() + { +#if NET5_0_OR_GREATER + const double HighPressureThreshold = .90; // Percent of GC memory pressure threshold we consider "high". + const double MediumPressureThreshold = .70; // Percent of GC memory pressure threshold we consider "medium". + + GCMemoryInfo memoryInfo = GC.GetGCMemoryInfo(); + + if (memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * HighPressureThreshold) + return MemoryPressure.High; + + if (memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * MediumPressureThreshold) + return MemoryPressure.Medium; + + return MemoryPressure.Low; +#else + return MemoryPressure.High; +#endif + } + + [DoesNotReturn] + public static void ThrowArgumentNullException_Obj() + => throw new ArgumentNullException("obj"); + + public static void ThrowArgumentOutOfRangeException_InitialColdCapacityCanNotBeNegative() + => throw new ArgumentOutOfRangeException("initialColdCapacity", "Can't be negative."); + + public static void ThrowArgumentOutOfRangeException_InitialCapacityCanNotBeNegative() + => throw new ArgumentOutOfRangeException("initialCapacity","Can't be negative."); + + public static void ThrowArgumentOutOfRangeException_HotCapacityCanNotBeLowerThanOne() + => throw new ArgumentOutOfRangeException("hotCapacity", "Can't be lower than 1."); + } +} \ No newline at end of file