UPM package version 3.3.0

CHANGELOG.md

@@ -1,6 +1,19 @@
 # Changelog
 All notable changes to this project will be documented in this file. Changes are listed in random order of importance.
 
+## [3.3.0] - 04-2022
+
+* INeedEGID and INeedEntityReference interfaces are not deprecated, but still available for backwards compatibility through the define SLOW_SVELTO_SUBMISSION
+* There are some minor breaking changes, you may need to rename a bunch of methods calls
+* Drastically improved Submission phase performance
+* All the IReactOn interfaces are now replaced by much faster IReacOn*Ex interfaces. Use those~~~~
+* QueryEntities methods now optionally return also an array of Entity IDs that you can reference like a component (this supersedes INeedEGID)
+* Completely reworked and way more powerful filter API. The old one has been renamed to Legacy and left for backward compatibility
+* NativeEGIDMultiMapper doesn't need to be created every submission anymore. It can be created permanently and disposed when not used anymore (some caveats with it)
+* Improved Serialization system
+* Improved SveltoOnDots system
+* Tons of other improvements and bug fixes
+~~~~
 ## [3.2.5]
 
 * refactor and improved NativeBag and UnsafeBlob. This fix a previously known crash with Unity IL2CPP

Components/EGIDComponent.cs

@@ -1,7 +1,9 @@
+#if SLOW_SVELTO_SUBMISSION
 namespace Svelto.ECS
 {
     public struct EGIDComponent:IEntityComponent, INeedEGID
     {
         public EGID ID { get; set; }
     }
-}
+}
+#endif

Components/EntityHierarchyComponent.cs

@@ -1,11 +0,0 @@
-namespace Svelto.ECS
-{
-    public struct EntityHierarchyComponent: IEntityComponent, INeedEGID
-    {
-        public readonly ExclusiveGroupStruct parentGroup;
-            
-        public EntityHierarchyComponent(ExclusiveGroup group): this() { parentGroup = group; }
-            
-        public EGID ID { get; set; }
-    }
-}

Components/EntityReferenceComponent.cs

@@ -1,11 +0,0 @@
-using Svelto.ECS.Reference;
-
-namespace Svelto.ECS
-{
-    //To do: this should be removed and the only reason it exists is to solve some edge case scenarios with
-    //the publish/consumer pattern
-    public struct EntityReferenceComponent:IEntityComponent, INeedEntityReference
-    {
-        public EntityReference selfReference { get; set; }
-    }
-}

Components/LinkedEntityComponent.cs

@@ -1,7 +0,0 @@
-namespace Svelto.ECS
-{
-    public struct LinkedEntityComponent : IEntityComponent 
-    {
-        public EGID linkedEntity;
-    }
-}

Core/CheckEntityUtilities.cs

@@ -4,7 +4,6 @@ using System.Diagnostics;
 #endif
 using System;
 using System.Collections.Generic;
-using System.Runtime.CompilerServices;
 using Svelto.DataStructures;
 
 namespace Svelto.ECS
@@ -16,9 +15,9 @@ namespace Svelto.ECS
     public partial class EnginesRoot
     {
 #if DONT_USE        
-        [Conditional("CHECK_ALL")]
+        [Conditional("MEANINGLESS")]
 #endif
-        void CheckRemoveEntityID(EGID egid, Type entityDescriptorType, [CallerMemberName] string caller = null)
+        void CheckRemoveEntityID(EGID egid, Type entityDescriptorType, string caller)
         {
             if (_multipleOperationOnSameEGIDChecker.ContainsKey(egid) == true)
                 throw new ECSException(
@@ -30,23 +29,29 @@ namespace Svelto.ECS
                        .FastConcat(_multipleOperationOnSameEGIDChecker[egid] == 1 ? "add" : "remove"));
 
             if (_idChecker.TryGetValue(egid.groupID, out var hash))
+            {
                 if (hash.Contains(egid.entityID) == false)
-                    throw new ECSException("Trying to remove an Entity never submitted in the database "
-                                          .FastConcat(" caller: ", caller, " ").FastConcat(egid.entityID)
-                                          .FastConcat(" groupid: ").FastConcat(egid.groupID.ToName())
-                                          .FastConcat(" type: ")
-                                          .FastConcat(entityDescriptorType != null
-                                                          ? entityDescriptorType.Name
-                                                          : "not available"));
-                else
-                    hash.Remove(egid.entityID);
+                    throw new ECSException("Trying to remove an Entity not present in the database "
+                       .FastConcat(" caller: ", caller, " entityID ").FastConcat(egid.entityID).FastConcat(" groupid: ")
+                       .FastConcat(egid.groupID.ToName()).FastConcat(" type: ")
+                       .FastConcat(entityDescriptorType != null ? entityDescriptorType.Name : "not available"));
+            }
+            else
+            {
+                throw new ECSException("Trying to remove an Entity with a group never used so far "
+                   .FastConcat(" caller: ", caller, " entityID ").FastConcat(egid.entityID).FastConcat(" groupid: ")
+                   .FastConcat(egid.groupID.ToName()).FastConcat(" type: ")
+                   .FastConcat(entityDescriptorType != null ? entityDescriptorType.Name : "not available"));
+            }
+
+            hash.Remove(egid.entityID);
 
             _multipleOperationOnSameEGIDChecker.Add(egid, 0);
         }
 #if DONT_USE
-        [Conditional("CHECK_ALL")]
+        [Conditional("MEANINGLESS")]
 #endif
-        void CheckAddEntityID(EGID egid, Type entityDescriptorType, [CallerMemberName] string caller = null)
+        void CheckAddEntityID(EGID egid, Type entityDescriptorType, string caller)
         {
             if (_multipleOperationOnSameEGIDChecker.ContainsKey(egid) == true)
                 throw new ECSException(
@@ -57,27 +62,31 @@ namespace Svelto.ECS
                        .FastConcat(" previous operation was: ")
                        .FastConcat(_multipleOperationOnSameEGIDChecker[egid] == 1 ? "add" : "remove"));
 
-            var hash = _idChecker.GetOrCreate(egid.groupID, () => new HashSet<uint>());
+            var hash = _idChecker.GetOrAdd(egid.groupID, () => new HashSet<uint>());
             if (hash.Contains(egid.entityID) == true)
-                throw new ECSException("Trying to add an Entity already submitted to the database "
-                                      .FastConcat(" caller: ", caller, " ").FastConcat(egid.entityID)
+                throw new ECSException("Trying to add an Entity already present in the database "
+                                      .FastConcat(" caller: ", caller, " entityID ").FastConcat(egid.entityID)
                                       .FastConcat(" groupid: ").FastConcat(egid.groupID.ToName()).FastConcat(" type: ")
                                       .FastConcat(entityDescriptorType != null
                                                       ? entityDescriptorType.Name
                                                       : "not available"));
             hash.Add(egid.entityID);
+            
             _multipleOperationOnSameEGIDChecker.Add(egid, 1);
         }
 
 #if DONT_USE
-        [Conditional("CHECK_ALL")]
+        [Conditional("MEANINGLESS")]
 #endif
-        void RemoveGroupID(ExclusiveBuildGroup groupID) { _idChecker.Remove(groupID); }
+        void RemoveGroupID(ExclusiveBuildGroup groupID)
+        {
+            _idChecker.Remove(groupID);
+        }
 
 #if DONT_USE
-        [Conditional("CHECK_ALL")]
+        [Conditional("MEANINGLESS")]
 #endif
-        void ClearChecks() { _multipleOperationOnSameEGIDChecker.FastClear(); }
+        void ClearDebugChecks() { _multipleOperationOnSameEGIDChecker.FastClear(); }
 
         readonly FasterDictionary<EGID, uint>                          _multipleOperationOnSameEGIDChecker;
         readonly FasterDictionary<ExclusiveGroupStruct, HashSet<uint>> _idChecker;

Core/ComponentBuilder.CheckFields.cs

@@ -4,10 +4,11 @@ using System.Diagnostics;
 #endif
 using System;
 using System.Reflection;
+using Svelto.ECS.Hybrid;
 
 namespace Svelto.ECS
 {
-    static class ComponentBuilderUtilities
+    public static class ComponentBuilderUtilities
     {
         const string MSG = "Entity Components and Entity View Components fields cannot hold managed fields outside the Svelto rules.";
 
@@ -63,16 +64,28 @@ namespace Svelto.ECS
 
                     for (int j = properties.Length - 1; j >= 0; --j)
                     {
-                        if (properties[j].PropertyType.IsGenericType)
+                        Type propertyType = properties[j].PropertyType;
+                        if (propertyType.IsGenericType)
                         {
-                            Type genericTypeDefinition = properties[j].PropertyType.GetGenericTypeDefinition();
+                            Type genericTypeDefinition = propertyType.GetGenericTypeDefinition();
                             if (genericTypeDefinition == RECATIVEVALUETYPE)
                             {
                                 continue;
                             }
                         }
 
-                        Type propertyType = properties[j].PropertyType;
+                        // Special rules for ValueReference<T>
+                        if (IsValueReference(propertyType))
+                        {
+                            // Getters of ValueReference must be refs, which would cause a failure on the common check.
+                            if (properties[j].CanRead == true && propertyType.IsByRef == false)
+                            {
+                                ProcessError(MSG, entityComponentType, propertyType);
+                            }
+
+                            continue;
+                        }
+
                         if (propertyType != STRINGTYPE)
                         {
                             //for EntityComponentStructs, component fields that are structs that hold strings
@@ -89,6 +102,12 @@ namespace Svelto.ECS
             return type == STRINGTYPE || type == STRINGBUILDERTYPE;
         }
 
+        static bool IsValueReference(Type type)
+        {
+            var interfaces = type.GetInterfaces();
+            return interfaces.Length == 1 && interfaces[0] == VALUE_REF_TYPE;
+        }
+
         /// <summary>
         /// This method checks the fields if it's an IEntityComponent, but checks all the properties if it's
         /// IEntityViewComponent
@@ -99,8 +118,9 @@ namespace Svelto.ECS
         static void SubCheckFields(Type fieldType, Type entityComponentType, bool isStringAllowed = false)
         {
             //pass if it's Primitive or C# 8 unmanaged, or it's a string and string are allowed
-            //this check must allow pointers are they are unmanaged types
-            if ((isStringAllowed == true && IsString(fieldType) == true) || fieldType.IsValueTypeEx() == true)
+            //this check must allow pointers as they are unmanaged types
+            if ((isStringAllowed == true && IsString(fieldType) == true) ||
+                fieldType.IsValueTypeEx() == true)
             {
                 //if it's a struct we have to check the fields recursively
                 if (IsString(fieldType) == false)
@@ -110,7 +130,7 @@ namespace Svelto.ECS
 
                 return;
             }
-            
+
             ProcessError(MSG, entityComponentType, fieldType);
         }
 
@@ -124,7 +144,8 @@ namespace Svelto.ECS
             throw new ECSException(message, entityComponentType);
         }
 
-        static readonly Type RECATIVEVALUETYPE       = typeof(ReactiveValue<>);
+        static readonly Type RECATIVEVALUETYPE          = typeof(ReactiveValue<>);
+        static readonly Type VALUE_REF_TYPE             = typeof(IValueReferenceInternal);
         static readonly Type EGIDType                   = typeof(EGID);
         static readonly Type EXCLUSIVEGROUPSTRUCTTYPE   = typeof(ExclusiveGroupStruct);
         static readonly Type SERIALIZABLE_ENTITY_STRUCT = typeof(SerializableEntityComponent);

Core/ComponentBuilder.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Reflection;
+using System.Threading;
 using DBC.ECS;
 using Svelto.Common;
 using Svelto.DataStructures;
@@ -22,33 +23,59 @@ namespace Svelto.ECS
             return obj.GetEntityComponentType().GetHashCode();
         }
     }
+
+    public static class BurstCompatibleCounter
+    {
+        public static int counter;        
+    }
     
-    public class ComponentBuilder<T> : IComponentBuilder
-        where T : struct, IEntityComponent
+    public class ComponentID<T> where T : struct, IEntityComponent
+    {
+        public static readonly SharedStaticWrapper<int, ComponentID<T>> id;
+
+#if UNITY_BURST 
+        [Unity.Burst.BurstDiscard] 
+        //SharedStatic values must be initialized from not burstified code
+#endif
+        public static void Init()
+        {
+            id.Data = Interlocked.Increment(ref BurstCompatibleCounter.counter);
+
+            DBC.ECS.Check.Ensure(id.Data < ushort.MaxValue, "too many types registered, HOW :)");
+        }
+    }
+
+    public class ComponentBuilder<T> : IComponentBuilder where T : struct, IEntityComponent
     {
         internal static readonly Type ENTITY_COMPONENT_TYPE;
-        public static readonly   bool HAS_EGID;
         internal static readonly bool IS_ENTITY_VIEW_COMPONENT;
 
         static readonly T      DEFAULT_IT;
         static readonly string ENTITY_COMPONENT_NAME;
         static readonly bool   IS_UNMANAGED;
-        public static   bool   HAS_REFERENCE;
+#if SLOW_SVELTO_SUBMISSION            
+        public static readonly bool HAS_EGID;
+        public static readonly bool HAS_REFERENCE;
+#endif
 
         static ComponentBuilder()
         {
-            ENTITY_COMPONENT_TYPE    = typeof(T);
-            DEFAULT_IT               = default;
+            ENTITY_COMPONENT_TYPE = typeof(T);
+            DEFAULT_IT = default;
             IS_ENTITY_VIEW_COMPONENT = typeof(IEntityViewComponent).IsAssignableFrom(ENTITY_COMPONENT_TYPE);
-            HAS_EGID                 = typeof(INeedEGID).IsAssignableFrom(ENTITY_COMPONENT_TYPE);
-            HAS_REFERENCE            = typeof(INeedEntityReference).IsAssignableFrom(ENTITY_COMPONENT_TYPE);
-            ENTITY_COMPONENT_NAME    = ENTITY_COMPONENT_TYPE.ToString();
-            IS_UNMANAGED             = ENTITY_COMPONENT_TYPE.IsUnmanagedEx();
-
+#if SLOW_SVELTO_SUBMISSION            
+            HAS_EGID = typeof(INeedEGID).IsAssignableFrom(ENTITY_COMPONENT_TYPE);
+            HAS_REFERENCE = typeof(INeedEntityReference).IsAssignableFrom(ENTITY_COMPONENT_TYPE);
+            
+            SetEGIDWithoutBoxing<T>.Warmup();
+#endif
+            ComponentID<T>.Init();
+            ENTITY_COMPONENT_NAME = ENTITY_COMPONENT_TYPE.ToString();
+            IS_UNMANAGED = TypeType.isUnmanaged<T>(); //attention this is important as it serves as warm up for Type<T>
+#if UNITY_NATIVE
             if (IS_UNMANAGED)
                 EntityComponentIDMap.Register<T>(new Filler<T>());
-
-            SetEGIDWithoutBoxing<T>.Warmup();
+#endif
 
             ComponentBuilderUtilities.CheckFields(ENTITY_COMPONENT_TYPE, IS_ENTITY_VIEW_COMPONENT);
 
@@ -58,16 +85,22 @@ namespace Svelto.ECS
             }
             else
             {
-                if (ENTITY_COMPONENT_TYPE != ComponentBuilderUtilities.ENTITY_INFO_COMPONENT
-                 && ENTITY_COMPONENT_TYPE.IsUnmanagedEx() == false)
+                if (ENTITY_COMPONENT_TYPE != ComponentBuilderUtilities.ENTITY_INFO_COMPONENT &&
+                    ENTITY_COMPONENT_TYPE.IsUnmanagedEx() == false)
                     throw new Exception(
                         $"Entity Component check failed, unexpected struct type (must be unmanaged) {ENTITY_COMPONENT_TYPE}");
             }
         }
 
-        public ComponentBuilder() { _initializer = DEFAULT_IT; }
+        public ComponentBuilder()
+        {
+            _initializer = DEFAULT_IT;
+        }
 
-        public ComponentBuilder(in T initializer) : this() { _initializer = initializer; }
+        public ComponentBuilder(in T initializer) : this()
+        {
+            _initializer = initializer;
+        }
 
         public bool isUnmanaged => IS_UNMANAGED;
 
@@ -75,37 +108,51 @@ namespace Svelto.ECS
         {
             var castedDic = dictionary as ITypeSafeDictionary<T>;
 
-            T entityComponent = default;
-
             if (IS_ENTITY_VIEW_COMPONENT)
             {
-                Check.Require(castedDic.ContainsKey(egid.entityID) == false
-                            , $"building an entity with already used entity id! id: '{(ulong) egid}', {ENTITY_COMPONENT_NAME}");
+                T entityComponent = default;
+                
+                Check.Require(castedDic.ContainsKey(egid.entityID) == false,
+                    $"building an entity with already used entity id! id: '{(ulong)egid}', {ENTITY_COMPONENT_NAME}");
 
-                this.SetEntityViewComponentImplementors(ref entityComponent, EntityViewComponentCache.cachedFields
-                                                      , implementors, EntityViewComponentCache.implementorsByType
-                                                      , EntityViewComponentCache.cachedTypes);
+                this.SetEntityViewComponentImplementors(ref entityComponent, EntityViewComponentCache.cachedFields,
+                    implementors, EntityViewComponentCache.implementorsByType, EntityViewComponentCache.cachedTypes);
 
                 castedDic.Add(egid.entityID, entityComponent);
             }
             else
             {
-                Check.Require(!castedDic.ContainsKey(egid.entityID)
-                            , $"building an entity with already used entity id! id: '{egid.entityID}'");
+                Check.Require(!castedDic.ContainsKey(egid.entityID),
+                    $"building an entity with already used entity id! id: '{egid.entityID}'");
 
                 castedDic.Add(egid.entityID, _initializer);
             }
         }
 
-        void IComponentBuilder.Preallocate(ITypeSafeDictionary dictionary, uint size) { Preallocate(dictionary, size); }
+        void IComponentBuilder.Preallocate(ITypeSafeDictionary dictionary, uint size)
+        {
+            Preallocate(dictionary, size);
+        }
 
-        public ITypeSafeDictionary CreateDictionary(uint size) { return TypeSafeDictionaryFactory<T>.Create(size); }
+        public ITypeSafeDictionary CreateDictionary(uint size)
+        {
+            return TypeSafeDictionaryFactory<T>.Create(size);
+        }
 
-        public Type GetEntityComponentType() { return ENTITY_COMPONENT_TYPE; }
+        public Type GetEntityComponentType()
+        {
+            return ENTITY_COMPONENT_TYPE;
+        }
 
-        public override int GetHashCode() { return _initializer.GetHashCode(); }
+        public override int GetHashCode()
+        {
+            return _initializer.GetHashCode();
+        }
 
-        static void Preallocate(ITypeSafeDictionary dictionary, uint size) { dictionary.ResizeTo(size); }
+        static void Preallocate(ITypeSafeDictionary dictionary, uint size)
+        {
+            dictionary.EnsureCapacity(size);
+        }
 
         readonly T _initializer;
 
@@ -156,7 +203,9 @@ namespace Svelto.ECS
 #endif
             }
 
-            internal static void InitCache() { }
+            internal static void InitCache()
+            {
+            }
         }
     }
 }

Core/DBC.cs

@@ -55,7 +55,7 @@ namespace DBC.ECS
     ///
     static class Check
     {
-        #region Interface
+#region Interface
 
         /// <summary>
         /// Precondition check.
@@ -65,15 +65,8 @@ namespace DBC.ECS
 #endif
         public static void Require(bool assertion, string message)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new PreconditionException(message);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Precondition: " + message);
-            }
+            if (!assertion)
+                throw new PreconditionException(message);
         }
 
         /// <summary>
@@ -85,15 +78,8 @@ namespace DBC.ECS
 #endif
         public static void Require(bool assertion, string message, Exception inner)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new PreconditionException(message, inner);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Precondition: " + message);
-            }
+            if (!assertion)
+                throw new PreconditionException(message, inner);
         }
 
         /// <summary>
@@ -105,15 +91,8 @@ namespace DBC.ECS
 #endif
         public static void Require(bool assertion)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new PreconditionException("Precondition failed.");
-            }
-            else
-            {
-                Trace.Assert(assertion, "Precondition failed.");
-            }
+            if (!assertion)
+                throw new PreconditionException("Precondition failed.");
         }
 
         /// <summary>
@@ -125,15 +104,8 @@ namespace DBC.ECS
 #endif
         public static void Ensure(bool assertion, string message)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new PostconditionException(message);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Postcondition: " + message);
-            }
+            if (!assertion)
+                throw new PostconditionException(message);
         }
 
         /// <summary>
@@ -145,15 +117,8 @@ namespace DBC.ECS
 #endif
         public static void Ensure(bool assertion, string message, Exception inner)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new PostconditionException(message, inner);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Postcondition: " + message);
-            }
+            if (!assertion)
+                throw new PostconditionException(message, inner);
         }
 
         /// <summary>
@@ -165,15 +130,8 @@ namespace DBC.ECS
 #endif
         public static void Ensure(bool assertion)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new PostconditionException("Postcondition failed.");
-            }
-            else
-            {
-                Trace.Assert(assertion, "Postcondition failed.");
-            }
+            if (!assertion)
+                throw new PostconditionException("Postcondition failed.");
         }
 
         /// <summary>
@@ -185,15 +143,8 @@ namespace DBC.ECS
 #endif
         public static void Invariant(bool assertion, string message)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new InvariantException(message);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Invariant: " + message);
-            }
+            if (!assertion)
+                throw new InvariantException(message);
         }
 
         /// <summary>
@@ -205,15 +156,8 @@ namespace DBC.ECS
 #endif
         public static void Invariant(bool assertion, string message, Exception inner)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new InvariantException(message, inner);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Invariant: " + message);
-            }
+            if (!assertion)
+                throw new InvariantException(message, inner);
         }
 
         /// <summary>
@@ -225,15 +169,8 @@ namespace DBC.ECS
 #endif
         public static void Invariant(bool assertion)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new InvariantException("Invariant failed.");
-            }
-            else
-            {
-                Trace.Assert(assertion, "Invariant failed.");
-            }
+            if (!assertion)
+                throw new InvariantException("Invariant failed.");
         }
 
         /// <summary>
@@ -244,15 +181,8 @@ namespace DBC.ECS
 #endif
         public static void Assert(bool assertion, string message)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new AssertionException(message);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Assertion: " + message);
-            }
+            if (!assertion)
+                throw new AssertionException(message);
         }
 
         /// <summary>
@@ -264,15 +194,8 @@ namespace DBC.ECS
 #endif
         public static void Assert(bool assertion, string message, Exception inner)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new AssertionException(message, inner);
-            }
-            else
-            {
-                Trace.Assert(assertion, "Assertion: " + message);
-            }
+            if (!assertion)
+                throw new AssertionException(message, inner);
         }
 
         /// <summary>
@@ -284,59 +207,22 @@ namespace DBC.ECS
 #endif
         public static void Assert(bool assertion)
         {
-            if (UseExceptions)
-            {
-                if (!assertion)
-                    throw new AssertionException("Assertion failed.");
-            }
-            else
-            {
-                Trace.Assert(assertion, "Assertion failed.");
-            }
+            if (!assertion)
+                throw new AssertionException("Assertion failed.");
         }
 
-        /// <summary>
-        /// Set this if you wish to use Trace Assert statements
-        /// instead of exception handling.
-        /// (The Check class uses exception handling by default.)
-        /// </summary>
-        public static bool UseAssertions
-        {
+#endregion // Interface
 
-            get
-            {
-                return useAssertions;
-            }
-            set
-            {
-                useAssertions = value;
-            }
-        }
-
-        #endregion // Interface
-
-        #region Implementation
+#region Implementation
 
         // No creation
 
         /// <summary>
         /// Is exception handling being used?
         /// </summary>
-        static bool UseExceptions
-        {
-            get
-            {
-                return !useAssertions;
-            }
-        }
-
-        // Are trace assertion statements being used?
-        // Default is to use exception handling.
-        static bool useAssertions;
-
-        #endregion // Implementation
 
-    } // End Check
+#endregion // Implementation
+    }      // End Check
 
     internal class Trace
     {
@@ -350,7 +236,7 @@ namespace DBC.ECS
         }
     }
 
-    #region Exceptions
+#region Exceptions
 
     /// <summary>
     /// Exception raised when a contract is broken.
@@ -360,9 +246,17 @@ namespace DBC.ECS
     /// </summary>
     public class DesignByContractException : Exception
     {
-        protected DesignByContractException() {}
-        protected DesignByContractException(string message) : base(message) {}
-        protected DesignByContractException(string message, Exception inner) : base(message, inner) {}
+        protected DesignByContractException()
+        {
+        }
+
+        protected DesignByContractException(string message) : base(message)
+        {
+        }
+
+        protected DesignByContractException(string message, Exception inner) : base(message, inner)
+        {
+        }
     }
 
     /// <summary>
@@ -373,15 +267,23 @@ namespace DBC.ECS
         /// <summary>
         /// Precondition Exception.
         /// </summary>
-        public PreconditionException() {}
+        public PreconditionException()
+        {
+        }
+
         /// <summary>
         /// Precondition Exception.
         /// </summary>
-        public PreconditionException(string message) : base(message) {}
+        public PreconditionException(string message) : base(message)
+        {
+        }
+
         /// <summary>
         /// Precondition Exception.
         /// </summary>
-        public PreconditionException(string message, Exception inner) : base(message, inner) {}
+        public PreconditionException(string message, Exception inner) : base(message, inner)
+        {
+        }
     }
 
     /// <summary>
@@ -392,15 +294,23 @@ namespace DBC.ECS
         /// <summary>
         /// Postcondition Exception.
         /// </summary>
-        public PostconditionException() {}
+        public PostconditionException()
+        {
+        }
+
         /// <summary>
         /// Postcondition Exception.
         /// </summary>
-        public PostconditionException(string message) : base(message) {}
+        public PostconditionException(string message) : base(message)
+        {
+        }
+
         /// <summary>
         /// Postcondition Exception.
         /// </summary>
-        public PostconditionException(string message, Exception inner) : base(message, inner) {}
+        public PostconditionException(string message, Exception inner) : base(message, inner)
+        {
+        }
     }
 
     /// <summary>
@@ -411,15 +321,23 @@ namespace DBC.ECS
         /// <summary>
         /// Invariant Exception.
         /// </summary>
-        public InvariantException() {}
+        public InvariantException()
+        {
+        }
+
         /// <summary>
         /// Invariant Exception.
         /// </summary>
-        public InvariantException(string message) : base(message) {}
+        public InvariantException(string message) : base(message)
+        {
+        }
+
         /// <summary>
         /// Invariant Exception.
         /// </summary>
-        public InvariantException(string message, Exception inner) : base(message, inner) {}
+        public InvariantException(string message, Exception inner) : base(message, inner)
+        {
+        }
     }
 
     /// <summary>
@@ -430,17 +348,24 @@ namespace DBC.ECS
         /// <summary>
         /// Assertion Exception.
         /// </summary>
-        public AssertionException() {}
+        public AssertionException()
+        {
+        }
+
         /// <summary>
         /// Assertion Exception.
         /// </summary>
-        public AssertionException(string message) : base(message) {}
+        public AssertionException(string message) : base(message)
+        {
+        }
+
         /// <summary>
         /// Assertion Exception.
         /// </summary>
-        public AssertionException(string message, Exception inner) : base(message, inner) {}
+        public AssertionException(string message, Exception inner) : base(message, inner)
+        {
+        }
     }
 
-    #endregion // Exception classes
-
-} // End Design By Contract
+#endregion // Exception classes
+}          // End Design By Contract

Core/EGIDMapper.cs

@@ -8,16 +8,16 @@ namespace Svelto.ECS
     /// <summary>
     /// </summary>
     /// <typeparam name="T"></typeparam>
-    public readonly struct EGIDMapper<T>: IEGIDMapper where T : struct, IEntityComponent
+    public readonly struct EGIDMapper<T> : IEGIDMapper where T : struct, IEntityComponent
     {
-        public uint                 length          => _map.count;
-        public ExclusiveGroupStruct groupID         { get; }
-        public Type                 entityType            => TypeCache<T>.type;
+        public int                  count      => _map.count;
+        public ExclusiveGroupStruct groupID    { get; }
+        public Type                 entityType => TypeCache<T>.type;
 
         internal EGIDMapper(ExclusiveGroupStruct groupStructId, ITypeSafeDictionary<T> dic) : this()
         {
             groupID = groupStructId;
-            _map     = dic;
+            _map    = dic;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -25,9 +25,10 @@ namespace Svelto.ECS
         {
 #if DEBUG && !PROFILE_SVELTO
             if (_map == null)
-                throw new System.Exception("Not initialized EGIDMapper in this group ".FastConcat(typeof(T).ToString()));
-                if (_map.TryFindIndex(entityID, out var findIndex) == false)
-                    throw new System.Exception("Entity not found in this group ".FastConcat(typeof(T).ToString()));
+                throw new System.Exception(
+                    "Not initialized EGIDMapper in this group ".FastConcat(typeof(T).ToString()));
+            if (_map.TryFindIndex(entityID, out var findIndex) == false)
+                throw new System.Exception("Entity not found in this group ".FastConcat(typeof(T).ToString()));
 #else
             _map.TryFindIndex(entityID, out var findIndex);
 #endif
@@ -72,8 +73,8 @@ namespace Svelto.ECS
         bool FindIndex(uint valueKey, out uint index);
         uint GetIndex(uint entityID);
         bool Exists(uint idEntityId);
-        
-        ExclusiveGroupStruct groupID { get; }
-        Type          entityType    { get; }
+
+        ExclusiveGroupStruct groupID    { get; }
+        Type                 entityType { get; }
     }
 }

Core/EnginesGroup/SortedEnginesGroup.cs

@@ -11,7 +11,6 @@ namespace Svelto.ECS
     ///{
     ///    WiresTimeRunningGroup,
     ///    WiresPreInitTimeRunningGroup,
-    ///    WiresInitTimeStoppedGroup,
     ///    WiresInitTimeRunningGroup
     ///}
     ///

Core/EnginesRoot.DoubleBufferedEntitiesToAdd.cs

@@ -7,77 +7,98 @@ namespace Svelto.ECS
     {
         internal class DoubleBufferedEntitiesToAdd
         {
-            const int MAX_NUMBER_OF_ITEMS_PER_FRAME_BEFORE_TO_CLEAR = 100;
+            //while caching is good to avoid over creating dictionaries that may be reused, the side effect
+            //is that I have to iterate every time up to 100 dictionaries during the flushing of the build entities
+            //even if there are 0 entities inside.
+            const int MAX_NUMBER_OF_GROUPS_TO_CACHE          = 100;
+            const int MAX_NUMBER_OF_TYPES_PER_GROUP_TO_CACHE = 100;
 
             public DoubleBufferedEntitiesToAdd()
             {
-                _currentEntitiesCreatedPerGroup = _entitiesCreatedPerGroupA;
-                _otherEntitiesCreatedPerGroup   = _entitiesCreatedPerGroupB;
+                var entitiesCreatedPerGroupA = new FasterDictionary<ExclusiveGroupStruct, uint>();
+                var entitiesCreatedPerGroupB = new FasterDictionary<ExclusiveGroupStruct, uint>();
+                var entityComponentsToAddBufferA =
+                    new FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>();
+                var entityComponentsToAddBufferB =
+                    new FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>();
 
-                current = _entityComponentsToAddBufferA;
-                other   = _entityComponentsToAddBufferB;
+                _currentNumberEntitiesCreatedPerGroup = entitiesCreatedPerGroupA;
+                _lastNumberEntitiesCreatedPerGroup   = entitiesCreatedPerGroupB;
+
+                currentComponentsToAddPerGroup = entityComponentsToAddBufferA;
+                lastComponentsToAddPerGroup   = entityComponentsToAddBufferB;
             }
 
-            public void ClearOther()
+            public void ClearLastAddOperations()
             {
-                //do not clear the groups created so far, they will be reused, unless they are too many!
-                var otherCount = other.count;
-                if (otherCount > MAX_NUMBER_OF_ITEMS_PER_FRAME_BEFORE_TO_CLEAR)
+                var numberOfGroupsAddedSoFar     = lastComponentsToAddPerGroup.count;
+                var componentDictionariesPerType = lastComponentsToAddPerGroup.unsafeValues;
+                
+                //TODO: rewrite the caching logic with the new RecycleOrAdd dictionary functionality
+                //I still do not want to cache too many groups
+                
+                //If we didn't create too many groups, we keep them alive, so we avoid the cost of creating new dictionaries
+                //during future submissions, otherwise we clean up everything
+                if (numberOfGroupsAddedSoFar > MAX_NUMBER_OF_GROUPS_TO_CACHE)
                 {
-                    var otherValuesArray = other.unsafeValues;
-                    for (var i = 0; i < otherCount; ++i)
+                    for (var i = 0; i < numberOfGroupsAddedSoFar; ++i)
                     {
-                        var safeDictionariesCount = otherValuesArray[i].count;
-                        var safeDictionaries      = otherValuesArray[i].unsafeValues;
+                        var componentTypesCount      = componentDictionariesPerType[i].count;
+                        var componentTypesDictionary = componentDictionariesPerType[i].unsafeValues;
                         {
-                            for (var j = 0; j < safeDictionariesCount; ++j)
-                                //clear the dictionary of entities create do far (it won't allocate though)
-                                safeDictionaries[j].Dispose();
+                            for (var j = 0; j < componentTypesCount; ++j)
+                                //dictionaries of components may be native so they need to be disposed
+                                //before the references are GCed
+                                componentTypesDictionary[j].Dispose();
                         }
                     }
-
+                
                     //reset the number of entities created so far
-                    _otherEntitiesCreatedPerGroup.FastClear();
-                    other.FastClear();
+                    _lastNumberEntitiesCreatedPerGroup.FastClear();
+                    lastComponentsToAddPerGroup.FastClear();
+                
                     return;
                 }
-
+                
+                for (var i = 0; i < numberOfGroupsAddedSoFar; ++i)
                 {
-                    var otherValuesArray = other.unsafeValues;
-                    for (var i = 0; i < otherCount; ++i)
+                    var                   componentTypesCount      = componentDictionariesPerType[i].count;
+                    ITypeSafeDictionary[] componentTypesDictionary = componentDictionariesPerType[i].unsafeValues;
+                    for (var j = 0; j < componentTypesCount; ++j)
+                        //clear the dictionary of entities created so far (it won't allocate though)
+                        componentTypesDictionary[j].Clear();
+                
+                    //if we didn't create too many component for this group, I reuse the component arrays
+                    if (componentTypesCount <= MAX_NUMBER_OF_TYPES_PER_GROUP_TO_CACHE)
                     {
-                        var safeDictionariesCount = otherValuesArray[i].count;
-                        var safeDictionaries      = otherValuesArray[i].unsafeValues;
-                        //do not remove the dictionaries of entities per type created so far, they will be reused
-                        if (safeDictionariesCount <= MAX_NUMBER_OF_ITEMS_PER_FRAME_BEFORE_TO_CLEAR)
-                        {
-                            for (var j = 0; j < safeDictionariesCount; ++j)
-                                //clear the dictionary of entities create do far (it won't allocate though)
-                                safeDictionaries[j].FastClear();
-                        }
-                        else
-                        {
-                            for (var j = 0; j < safeDictionariesCount; ++j)
-                                //clear the dictionary of entities create do far (it won't allocate though)
-                                safeDictionaries[j].Dispose();
-
-                            otherValuesArray[i].FastClear();
-                        }
+                        for (var j = 0; j < componentTypesCount; ++j)
+                            componentTypesDictionary[j].Clear();
+                    }
+                    else
+                    {
+                        //here I have to dispose, because I am actually clearing the reference of the dictionary
+                        //with the next line.
+                        for (var j = 0; j < componentTypesCount; ++j)
+                            componentTypesDictionary[j].Dispose();
+                
+                        componentDictionariesPerType[i].FastClear();
                     }
-
-                    //reset the number of entities created so far
-                    _otherEntitiesCreatedPerGroup.FastClear();
                 }
+
+                //reset the number of entities created so far
+                _lastNumberEntitiesCreatedPerGroup.FastClear();
+
+          //      _totalEntitiesToAdd = 0;
             }
 
             public void Dispose()
             {
                 {
-                    var otherValuesArray = other.unsafeValues;
-                    for (var i = 0; i < other.count; ++i)
+                    var otherValuesArray = lastComponentsToAddPerGroup.unsafeValues;
+                    for (var i = 0; i < lastComponentsToAddPerGroup.count; ++i)
                     {
-                        var safeDictionariesCount = otherValuesArray[i].count;
-                        var safeDictionaries      = otherValuesArray[i].unsafeValues;
+                        int                   safeDictionariesCount = otherValuesArray[i].count;
+                        ITypeSafeDictionary[] safeDictionaries      = otherValuesArray[i].unsafeValues;
                         //do not remove the dictionaries of entities per type created so far, they will be reused
                         for (var j = 0; j < safeDictionariesCount; ++j)
                             //clear the dictionary of entities create do far (it won't allocate though)
@@ -85,98 +106,158 @@ namespace Svelto.ECS
                     }
                 }
                 {
-                    var currentValuesArray = current.unsafeValues;
-                    for (var i = 0; i < current.count; ++i)
+                    var currentValuesArray = currentComponentsToAddPerGroup.unsafeValues;
+                    for (var i = 0; i < currentComponentsToAddPerGroup.count; ++i)
                     {
-                        var safeDictionariesCount = currentValuesArray[i].count;
-                        var safeDictionaries      = currentValuesArray[i].unsafeValues;
+                        int                   safeDictionariesCount = currentValuesArray[i].count;
+                        ITypeSafeDictionary[] safeDictionaries      = currentValuesArray[i].unsafeValues;
                         //do not remove the dictionaries of entities per type created so far, they will be reused
                         for (var j = 0; j < safeDictionariesCount; ++j)
                             //clear the dictionary of entities create do far (it won't allocate though)
                             safeDictionaries[j].Dispose();
                     }
                 }
+
+                _currentNumberEntitiesCreatedPerGroup = null;
+                _lastNumberEntitiesCreatedPerGroup   = null;
+                lastComponentsToAddPerGroup          = null;
+                currentComponentsToAddPerGroup        = null;
             }
 
             internal bool AnyEntityCreated()
             {
-                return _currentEntitiesCreatedPerGroup.count > 0;
+                return _currentNumberEntitiesCreatedPerGroup.count > 0;
             }
 
-            internal bool AnyOtherEntityCreated()
+            internal bool AnyPreviousEntityCreated()
             {
-                return _otherEntitiesCreatedPerGroup.count > 0;
+                return _lastNumberEntitiesCreatedPerGroup.count > 0;
             }
 
             internal void IncrementEntityCount(ExclusiveGroupStruct groupID)
             {
-                _currentEntitiesCreatedPerGroup.GetOrCreate(groupID)++;
+                _currentNumberEntitiesCreatedPerGroup.GetOrAdd(groupID)++;
+             //   _totalEntitiesToAdd++;
             }
 
+            // public uint NumberOfEntitiesToAdd()
+            // {
+            //     return _totalEntitiesToAdd;
+            // }
+
             internal void Preallocate
                 (ExclusiveGroupStruct groupID, uint numberOfEntities, IComponentBuilder[] entityComponentsToBuild)
             {
                 void PreallocateDictionaries
                     (FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>> dic)
                 {
-                    var group = dic.GetOrCreate(groupID, () => new FasterDictionary<RefWrapperType,
-                                                                      ITypeSafeDictionary>());
+                    var group = dic.GetOrAdd(
+                        groupID, () => new FasterDictionary<RefWrapperType, ITypeSafeDictionary>());
 
                     foreach (var componentBuilder in entityComponentsToBuild)
                     {
                         var entityComponentType = componentBuilder.GetEntityComponentType();
-                        var safeDictionary = group.GetOrCreate(new RefWrapperType(entityComponentType)
+                        var safeDictionary = group.GetOrAdd(new RefWrapperType(entityComponentType)
                                                              , () => componentBuilder
                                                                   .CreateDictionary(numberOfEntities));
                         componentBuilder.Preallocate(safeDictionary, numberOfEntities);
                     }
                 }
 
-                PreallocateDictionaries(current);
-                PreallocateDictionaries(other);
+                PreallocateDictionaries(currentComponentsToAddPerGroup);
+                PreallocateDictionaries(lastComponentsToAddPerGroup);
 
-                _currentEntitiesCreatedPerGroup.GetOrCreate(groupID);
-                _otherEntitiesCreatedPerGroup.GetOrCreate(groupID);
+                _currentNumberEntitiesCreatedPerGroup.GetOrAdd(groupID);
+                _lastNumberEntitiesCreatedPerGroup.GetOrAdd(groupID);
             }
 
             internal void Swap()
             {
-                Swap(ref current, ref other);
-                Swap(ref _currentEntitiesCreatedPerGroup, ref _otherEntitiesCreatedPerGroup);
+                Swap(ref currentComponentsToAddPerGroup, ref lastComponentsToAddPerGroup);
+                Swap(ref _currentNumberEntitiesCreatedPerGroup, ref _lastNumberEntitiesCreatedPerGroup);
             }
 
-            void Swap<T>(ref T item1, ref T item2)
+            static void Swap<T>(ref T item1, ref T item2)
             {
-                var toSwap = item2;
-                item2 = item1;
-                item1 = toSwap;
+                (item2, item1) = (item1, item2);
+            }
+
+            public OtherComponentsToAddPerGroupEnumerator GetEnumerator()
+            {
+                return new OtherComponentsToAddPerGroupEnumerator(lastComponentsToAddPerGroup
+                                                                , _lastNumberEntitiesCreatedPerGroup);
             }
 
             //Before I tried for the third time to use a SparseSet instead of FasterDictionary, remember that
             //while group indices are sequential, they may not be used in a sequential order. Sparseset needs
             //entities to be created sequentially (the index cannot be managed externally)
-            internal FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>> current;
-            internal FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>> other;
-
-            readonly FasterDictionary<ExclusiveGroupStruct, uint> _entitiesCreatedPerGroupA =
-                new FasterDictionary<ExclusiveGroupStruct, uint>();
-
-            readonly FasterDictionary<ExclusiveGroupStruct, uint> _entitiesCreatedPerGroupB =
-                new FasterDictionary<ExclusiveGroupStruct, uint>();
-
-            readonly FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>> _entityComponentsToAddBufferA =
-                    new FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>();
+            internal FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>
+                currentComponentsToAddPerGroup;
 
-            readonly FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>> _entityComponentsToAddBufferB =
-                    new FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>();
+            FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>
+                lastComponentsToAddPerGroup;
 
             /// <summary>
             ///     To avoid extra allocation, I don't clear the groups, so I need an extra data structure
             ///     to keep count of the number of entities built this frame. At the moment the actual number
             ///     of entities built is not used
             /// </summary>
-            FasterDictionary<ExclusiveGroupStruct, uint> _currentEntitiesCreatedPerGroup;
-            FasterDictionary<ExclusiveGroupStruct, uint> _otherEntitiesCreatedPerGroup;
+            FasterDictionary<ExclusiveGroupStruct, uint> _currentNumberEntitiesCreatedPerGroup;
+            FasterDictionary<ExclusiveGroupStruct, uint> _lastNumberEntitiesCreatedPerGroup;
+
+            //uint _totalEntitiesToAdd;
         }
     }
+
+    struct OtherComponentsToAddPerGroupEnumerator
+    {
+        public OtherComponentsToAddPerGroupEnumerator
+        (FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>
+             lastComponentsToAddPerGroup
+       , FasterDictionary<ExclusiveGroupStruct, uint> otherNumberEntitiesCreatedPerGroup)
+        {
+            _lastComponentsToAddPerGroup       = lastComponentsToAddPerGroup;
+            _lastNumberEntitiesCreatedPerGroup = otherNumberEntitiesCreatedPerGroup.GetEnumerator();
+            Current                             = default;
+        }
+
+        public bool MoveNext()
+        {
+            while (_lastNumberEntitiesCreatedPerGroup.MoveNext())
+            {
+                var current = _lastNumberEntitiesCreatedPerGroup.Current;
+
+                if (current.value > 0) //there are entities in this group
+                {
+                    var value = _lastComponentsToAddPerGroup[current.key];
+                    Current = new GroupInfo()
+                    {
+                        group      = current.key
+                      , components = value
+                    };
+
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        public GroupInfo Current { get; private set; }
+
+        //cannot be read only as they will be modified by MoveNext
+        readonly FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>
+            _lastComponentsToAddPerGroup;
+
+        SveltoDictionaryKeyValueEnumerator<ExclusiveGroupStruct, uint,
+                ManagedStrategy<SveltoDictionaryNode<ExclusiveGroupStruct>>, ManagedStrategy<uint>,
+                ManagedStrategy<int>>
+            _lastNumberEntitiesCreatedPerGroup;
+    }
+
+    struct GroupInfo
+    {
+        public ExclusiveGroupStruct                                  group;
+        public FasterDictionary<RefWrapperType, ITypeSafeDictionary> components;
+    }
 }

Core/EnginesRoot.Engines.cs

@@ -1,19 +1,33 @@
+#if PROFILE_SVELTO && DEBUG
+#warning the global define PROFILE_SVELTO should be used only when it's necessary to profile in order to reduce the overhead of debug code. Normally remove this define to get insights when errors happen
+#endif
+
 using System;
 using System.Collections.Generic;
+using DBC.ECS;
 using Svelto.Common;
+using Svelto.Common.DataStructures;
 using Svelto.DataStructures;
 using Svelto.ECS.Internal;
 using Svelto.ECS.Schedulers;
 
 namespace Svelto.ECS
 {
-    public sealed partial class EnginesRoot
+    public partial class EnginesRoot
     {
         static EnginesRoot()
         {
             GroupHashMap.Init();
+            SharedDictonary.Init();
             SerializationDescriptorMap.Init();
+
+
+            _swapEntities   = SwapEntities;
+            _removeEntities = RemoveEntities;
+            _removeGroup    = RemoveGroup;
+            _swapGroup      = SwapGroup;
         }
+
         /// <summary>
         ///     Engines root contextualize your engines and entities. You don't need to limit yourself to one EngineRoot
         ///     as multiple engines root could promote separation of scopes. The EntitySubmissionScheduler checks
@@ -24,36 +38,51 @@ namespace Svelto.ECS
         /// </summary>
         public EnginesRoot(EntitiesSubmissionScheduler entitiesComponentScheduler)
         {
-            _entitiesOperations                 = new FasterDictionary<ulong, EntitySubmitOperation>();
-            _idChecker                          = new FasterDictionary<ExclusiveGroupStruct, HashSet<uint>>();
-            _multipleOperationOnSameEGIDChecker = new FasterDictionary<EGID, uint>();
-#if UNITY_NATIVE //because of the thread count, ATM this is only for unity            
-            _nativeSwapOperationQueue   = new DataStructures.AtomicNativeBags(Allocator.Persistent);
-            _nativeRemoveOperationQueue = new DataStructures.AtomicNativeBags(Allocator.Persistent);
-            _nativeAddOperationQueue    = new DataStructures.AtomicNativeBags(Allocator.Persistent);
-#endif            
-            _serializationDescriptorMap     = new SerializationDescriptorMap();
-            _maxNumberOfOperationsPerFrame = uint.MaxValue;
-            _reactiveEnginesAddRemove      = new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>>();
-            _reactiveEnginesAddRemoveOnDispose =
-                new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>>();
-            _reactiveEnginesSwap         = new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>>();
-            _reactiveEnginesSubmission   = new FasterList<IReactOnSubmission>();
-            _enginesSet                  = new FasterList<IEngine>();
-            _enginesTypeSet              = new HashSet<Type>();
-            _disposableEngines           = new FasterList<IDisposable>();
-            _transientEntitiesOperations = new FasterList<EntitySubmitOperation>();
+            _entitiesOperations                  = new EntitiesOperations();
+            _idChecker                           = new FasterDictionary<ExclusiveGroupStruct, HashSet<uint>>();
+
+            _cachedRangeOfSubmittedIndices                 = new FasterList<(uint, uint)>();
+            _cachedIndicesToSwapBeforeSubmissionForFilters = new FasterDictionary<uint, uint>();
+            
+            _multipleOperationOnSameEGIDChecker            = new FasterDictionary<EGID, uint>();
+#if UNITY_NATIVE //because of the thread count, ATM this is only for unity
+            _nativeSwapOperationQueue   = new Svelto.ECS.DataStructures.AtomicNativeBags(Allocator.Persistent);
+            _nativeRemoveOperationQueue = new Svelto.ECS.DataStructures.AtomicNativeBags(Allocator.Persistent);
+            _nativeAddOperationQueue    = new Svelto.ECS.DataStructures.AtomicNativeBags(Allocator.Persistent);
+#endif
+            _serializationDescriptorMap = new SerializationDescriptorMap();
+            _reactiveEnginesAdd = new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnAdd>>>();
+            _reactiveEnginesAddEx =
+                new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnAddEx>>>();
+            _reactiveEnginesRemove =
+                new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnRemove>>>();
+            _reactiveEnginesRemoveEx =
+                new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnRemoveEx>>>();
+            _reactiveEnginesSwap =
+                new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnSwap>>>();
+            _reactiveEnginesSwapEx =
+                new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnSwapEx>>>();
+
+            _reactiveEnginesDispose =
+                new FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnDispose>>>();
+
+            _reactiveEnginesSubmission = new FasterList<IReactOnSubmission>();
+            _enginesSet                = new FasterList<IEngine>();
+            _enginesTypeSet            = new HashSet<Type>();
+            _disposableEngines         = new FasterList<IDisposable>();
 
             _groupEntityComponentsDB =
                 new FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>();
             _groupsPerEntity =
                 new FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, ITypeSafeDictionary>>();
             _groupedEntityToAdd = new DoubleBufferedEntitiesToAdd();
-            _entityStreams = EntitiesStreams.Create();
+            _entityStreams      = EntitiesStreams.Create();
             _groupFilters =
-                new FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, GroupFilters>>();
+                new FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, LegacyGroupFilters>>();
             _entityLocator.InitEntityReferenceMap();
-            _entitiesDB = new EntitiesDB(this,_entityLocator);
+            _entitiesDB = new EntitiesDB(this, _entityLocator);
+
+            InitFilters();
 
             scheduler        = entitiesComponentScheduler;
             scheduler.onTick = new EntitiesSubmitter(this);
@@ -62,11 +91,10 @@ namespace Svelto.ECS
 #endif
         }
 
-        public EnginesRoot
-            (EntitiesSubmissionScheduler entitiesComponentScheduler, bool isDeserializationOnly) :
-            this(entitiesComponentScheduler)
+        protected EnginesRoot(EntitiesSubmissionScheduler entitiesComponentScheduler,
+            EnginesReadyOption enginesWaitForReady) : this(entitiesComponentScheduler)
         {
-            _isDeserializationOnly = isDeserializationOnly;
+            _enginesWaitForReady = enginesWaitForReady;
         }
 
         public EntitiesSubmissionScheduler scheduler { get; }
@@ -78,7 +106,117 @@ namespace Svelto.ECS
         /// </summary>
         public void Dispose()
         {
-            _isDisposing = true;
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        public void AddEngine(IEngine engine)
+        {
+            var type       = engine.GetType();
+            var refWrapper = new RefWrapperType(type);
+            Check.Require(engine != null, "Engine to add is invalid or null");
+            Check.Require(
+                _enginesTypeSet.Contains(refWrapper) == false ||
+                type.ContainsCustomAttribute(typeof(AllowMultipleAttribute)),
+                "The same engine has been added more than once, if intentional, use [AllowMultiple] class attribute "
+                   .FastConcat(engine.ToString()));
+            try
+            {
+                if (engine is IReactOnAdd viewEngineAdd)
+                    CheckReactEngineComponents(typeof(IReactOnAdd<>), viewEngineAdd, _reactiveEnginesAdd, type.Name);
+
+                if (engine is IReactOnAddEx viewEngineAddEx)
+                    CheckReactEngineComponents(typeof(IReactOnAddEx<>), viewEngineAddEx, _reactiveEnginesAddEx, type.Name);
+
+                if (engine is IReactOnRemove viewEngineRemove)
+                    CheckReactEngineComponents(typeof(IReactOnRemove<>), viewEngineRemove, _reactiveEnginesRemove, type.Name);
+
+                if (engine is IReactOnRemoveEx viewEngineRemoveEx)
+                    CheckReactEngineComponents(typeof(IReactOnRemoveEx<>), viewEngineRemoveEx, _reactiveEnginesRemoveEx, type.Name);
+
+                if (engine is IReactOnDispose viewEngineDispose)
+                    CheckReactEngineComponents(typeof(IReactOnDispose<>), viewEngineDispose, _reactiveEnginesDispose, type.Name);
+
+                if (engine is IReactOnSwap viewEngineSwap)
+                    CheckReactEngineComponents(typeof(IReactOnSwap<>), viewEngineSwap, _reactiveEnginesSwap, type.Name);
+
+                if (engine is IReactOnSwapEx viewEngineSwapEx)
+                    CheckReactEngineComponents(typeof(IReactOnSwapEx<>), viewEngineSwapEx, _reactiveEnginesSwapEx, type.Name);
+
+                if (engine is IReactOnSubmission submissionEngine)
+                    _reactiveEnginesSubmission.Add(submissionEngine);
+
+                _enginesTypeSet.Add(refWrapper);
+                _enginesSet.Add(engine);
+
+                if (engine is IDisposable)
+                    _disposableEngines.Add(engine as IDisposable);
+
+                if (engine is IQueryingEntitiesEngine queryableEntityComponentEngine)
+                    queryableEntityComponentEngine.entitiesDB = _entitiesDB;
+
+                if (_enginesWaitForReady == EnginesReadyOption.ReadyAsAdded && engine is IGetReadyEngine getReadyEngine)
+                    getReadyEngine.Ready();
+            }
+            catch (Exception e)
+            {
+                throw new ECSException("Code crashed while adding engine ".FastConcat(engine.GetType().ToString(), " "),
+                    e);
+            }
+        }
+
+        public void Ready()
+        {
+            Check.Require(_enginesWaitForReady == EnginesReadyOption.WaitForReady,
+                "The engine has not been initialise to wait for an external ready trigger");
+
+            foreach (var engine in _enginesSet)
+                if (engine is IGetReadyEngine getReadyEngine)
+                    getReadyEngine.Ready();
+        }
+
+        static void AddEngineToList<T>(T engine, Type[] entityComponentTypes,
+            FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<T>>> engines, string typeName)
+            where T : class, IReactEngine
+        {
+            for (var i = 0; i < entityComponentTypes.Length; i++)
+            {
+                var type = entityComponentTypes[i];
+
+                if (engines.TryGetValue(new RefWrapperType(type), out var list) == false)
+                {
+                    list = new FasterList<ReactEngineContainer<T>>();
+
+                    engines.Add(new RefWrapperType(type), list);
+                }
+
+                list.Add(new ReactEngineContainer<T>(engine, typeName));
+            }
+        }
+
+        void CheckReactEngineComponents<T>(Type genericDefinition, T engine,
+            FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<T>>> engines, string typeName)
+            where T : class, IReactEngine
+        {
+            var interfaces = engine.GetType().GetInterfaces();
+
+            foreach (var interf in interfaces)
+            {
+                if (interf.IsGenericTypeEx() && interf.GetGenericTypeDefinition() == genericDefinition)
+                {
+                    var genericArguments = interf.GetGenericArgumentsEx();
+
+                    AddEngineToList(engine, genericArguments, engines, typeName);
+                }
+            }
+        }
+
+        void Dispose(bool disposing)
+        {
+            _isDisposing = disposing;
+
+            if (disposing == false)
+                return;
 
             using (var profiler = new PlatformProfiler("Final Dispose"))
             {
@@ -99,11 +237,13 @@ namespace Svelto.ECS
                     }
 
                 foreach (var groups in _groupEntityComponentsDB)
-                foreach (var entityList in groups.Value)
+                foreach (var entityList in groups.value)
                     try
                     {
-                        entityList.Value.ExecuteEnginesRemoveCallbacks(_reactiveEnginesAddRemoveOnDispose, profiler
-                                                                     , groups.Key);
+                        ITypeSafeDictionary typeSafeDictionary = entityList.value;
+                        
+                        typeSafeDictionary.ExecuteEnginesDisposeCallbacks_Group(_reactiveEnginesDispose, groups.key,
+                            profiler);
                     }
                     catch (Exception e)
                     {
@@ -111,15 +251,17 @@ namespace Svelto.ECS
                     }
 
                 foreach (var groups in _groupEntityComponentsDB)
-                foreach (var entityList in groups.Value)
-                    entityList.Value.Dispose();
+                foreach (var entityList in groups.value)
+                    entityList.value.Dispose();
 
                 foreach (var type in _groupFilters)
-                foreach (var group in type.Value)
-                    group.Value.Dispose();
+                foreach (var group in type.value)
+                    group.value.Dispose();
 
                 _groupFilters.Clear();
 
+                DisposeFilters();
+
 #if UNITY_NATIVE
                 _nativeAddOperationQueue.Dispose();
                 _nativeRemoveOperationQueue.Dispose();
@@ -132,207 +274,128 @@ namespace Svelto.ECS
                 _enginesSet.Clear();
                 _enginesTypeSet.Clear();
                 _reactiveEnginesSwap.Clear();
-                _reactiveEnginesAddRemove.Clear();
-                _reactiveEnginesAddRemoveOnDispose.Clear();
+                _reactiveEnginesAdd.Clear();
+                _reactiveEnginesRemove.Clear();
+                _reactiveEnginesDispose.Clear();
                 _reactiveEnginesSubmission.Clear();
 
-                _entitiesOperations.Clear();
-                _transientEntitiesOperations.Clear();
-
                 _groupedEntityToAdd.Dispose();
 
                 _entityLocator.DisposeEntityReferenceMap();
-                
+
                 _entityStreams.Dispose();
                 scheduler.Dispose();
             }
+        }
 
-            GC.SuppressFinalize(this);
+        void NotifyReactiveEnginesOnSubmission()
+        {
+            var enginesCount = _reactiveEnginesSubmission.count;
+            for (var i = 0; i < enginesCount; i++)
+                _reactiveEnginesSubmission[i].EntitiesSubmitted();
         }
 
-        public void AddEngine(IEngine engine)
+        public readonly struct EntitiesSubmitter
         {
-            var type       = engine.GetType();
-            var refWrapper = new RefWrapperType(type);
-            DBC.ECS.Check.Require(engine != null, "Engine to add is invalid or null");
-            DBC.ECS.Check.Require(
-                _enginesTypeSet.Contains(refWrapper) == false
-             || type.ContainsCustomAttribute(typeof(AllowMultipleAttribute)) == true
-              , "The same engine has been added more than once, if intentional, use [AllowMultiple] class attribute "
-                   .FastConcat(engine.ToString()));
-            try
+            public EntitiesSubmitter(EnginesRoot enginesRoot) : this()
             {
-                if (engine is IReactOnAddAndRemove viewEngine)
-                    CheckReactEngineComponents(viewEngine, _reactiveEnginesAddRemove, type.Name);
+                _enginesRoot = new Svelto.DataStructures.WeakReference<EnginesRoot>(enginesRoot);
+            }
 
-                if (engine is IReactOnDispose viewEngineDispose)
-                    CheckReactEngineComponents(viewEngineDispose, _reactiveEnginesAddRemoveOnDispose, type.Name);
+            internal void SubmitEntities()
+            {
+                Check.Require(_enginesRoot.IsValid, "ticking an GCed engines root?");
 
-                if (engine is IReactOnSwap viewEngineSwap)
-                    CheckReactEngineComponents(viewEngineSwap, _reactiveEnginesSwap, type.Name);
+                var enginesRootTarget           = _enginesRoot.Target;
+                var entitiesSubmissionScheduler = enginesRootTarget.scheduler;
 
-                if (engine is IReactOnSubmission submissionEngine)
-                    _reactiveEnginesSubmission.Add(submissionEngine);
+                if (entitiesSubmissionScheduler.paused == false)
+                {
+                    Check.Require(entitiesSubmissionScheduler.isRunning == false,
+                        "A submission started while the previous one was still flushing");
+                    entitiesSubmissionScheduler.isRunning = true;
 
-                _enginesTypeSet.Add(refWrapper);
-                _enginesSet.Add(engine);
+                    using (var profiler = new PlatformProfiler("Svelto.ECS - Entities Submission"))
+                    {
+                        var iterations       = 0;
+                        var hasEverSubmitted = false;
 
-                if (engine is IDisposable)
-                    _disposableEngines.Add(engine as IDisposable);
+                        // We need to clear transient filters before processing callbacks since the callbacks may add
+                        // new entities to these filters.
+                        enginesRootTarget.ClearTransientFilters();
 
-                if (engine is IQueryingEntitiesEngine queryableEntityComponentEngine)
-                {
-                    queryableEntityComponentEngine.entitiesDB = _entitiesDB;
-                    queryableEntityComponentEngine.Ready();
+#if UNITY_NATIVE
+                        enginesRootTarget.FlushNativeOperations(profiler);
+#endif
+                        //todo: proper unit test structural changes made as result of add/remove callbacks
+                        while (enginesRootTarget.HasMadeNewStructuralChangesInThisIteration() 
+                               && iterations++ < MAX_SUBMISSION_ITERATIONS)
+                        {
+                            hasEverSubmitted = true;
+
+                            _enginesRoot.Target.SingleSubmission(profiler);
+#if UNITY_NATIVE
+                            if (enginesRootTarget.HasMadeNewStructuralChangesInThisIteration())
+                                enginesRootTarget.FlushNativeOperations(profiler);
+#endif
+                        }
+
+#if DEBUG && !PROFILE_SVELTO
+                        if (iterations == MAX_SUBMISSION_ITERATIONS)
+                            throw new ECSException("possible circular submission detected");
+#endif
+                        if (hasEverSubmitted)
+                            enginesRootTarget.NotifyReactiveEnginesOnSubmission();
+                    }
+
+                    entitiesSubmissionScheduler.isRunning = false;
+                    ++entitiesSubmissionScheduler.iteration;
                 }
             }
-            catch (Exception e)
-            {
-                throw new ECSException("Code crashed while adding engine ".FastConcat(engine.GetType().ToString(), " ")
-                                     , e);
-            }
-        }
 
-        void NotifyReactiveEnginesOnSubmission()
-        {
-            var enginesCount = _reactiveEnginesSubmission.count;
-            for (var i = 0; i < enginesCount; i++)
-                _reactiveEnginesSubmission[i].EntitiesSubmitted();
+            readonly Svelto.DataStructures.WeakReference<EnginesRoot> _enginesRoot;
         }
 
         ~EnginesRoot()
         {
             Console.LogWarning("Engines Root has been garbage collected, don't forget to call Dispose()!");
 
-            Dispose();
+            Dispose(false);
         }
 
-        void CheckReactEngineComponents<T>
-            (T engine, FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> engines, string typeName)
-            where T : class, IReactEngine
-        {
-            var interfaces = engine.GetType().GetInterfaces();
-
-            foreach (var interf in interfaces)
-                if (interf.IsGenericTypeEx() && typeof(T).IsAssignableFrom(interf))
-                {
-                    var genericArguments = interf.GetGenericArgumentsEx();
-
-                    AddEngineToList(engine, genericArguments, engines, typeName);
-                }
-        }
-
-        static void AddEngineToList<T>
-        (T engine, Type[] entityComponentTypes
-       , FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> engines, string typeName)
-            where T : class, IReactEngine
-        {
-            for (var i = 0; i < entityComponentTypes.Length; i++)
-            {
-                var type = entityComponentTypes[i];
-
-                if (engines.TryGetValue(new RefWrapperType(type), out var list) == false)
-                {
-                    list = new FasterList<ReactEngineContainer>();
-
-                    engines.Add(new RefWrapperType(type), list);
-                }
-
-                list.Add(new ReactEngineContainer(engine, typeName));
-            }
-        }
+        const int MAX_SUBMISSION_ITERATIONS = 10;
 
         internal bool                    _isDisposing;
         readonly FasterList<IDisposable> _disposableEngines;
         readonly FasterList<IEngine>     _enginesSet;
         readonly HashSet<Type>           _enginesTypeSet;
+        readonly EnginesReadyOption      _enginesWaitForReady;
 
-        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> _reactiveEnginesAddRemove;
-        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> _reactiveEnginesAddRemoveOnDispose;
-        readonly FasterList<IReactOnSubmission>                                     _reactiveEnginesSubmission;
-        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> _reactiveEnginesSwap;
-
-        public struct EntitiesSubmitter
-        {
-            public EntitiesSubmitter(EnginesRoot enginesRoot) : this()
-            {
-                _enginesRoot = new Svelto.DataStructures.WeakReference<EnginesRoot>(enginesRoot);
-                _privateSubmitEntities =
-                    _enginesRoot.Target.SingleSubmission(new PlatformProfiler());
-                submitEntities = Invoke(); //this must be last to capture all the variables
-            }
-
-            IEnumerator<bool> Invoke()
-            {
-                while (true)
-                {
-                    DBC.ECS.Check.Require(_enginesRoot.IsValid, "ticking an GCed engines root?");
+        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnAdd>>> _reactiveEnginesAdd;
 
-                    var enginesRootTarget           = _enginesRoot.Target;
-                    var entitiesSubmissionScheduler = enginesRootTarget.scheduler;
+        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnAddEx>>>
+            _reactiveEnginesAddEx;
 
-                    if (entitiesSubmissionScheduler.paused == false)
-                    {
-                        DBC.ECS.Check.Require(entitiesSubmissionScheduler.isRunning == false
-                                            , "A submission started while the previous one was still flushing");
-                        entitiesSubmissionScheduler.isRunning = true;
+        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnRemove>>>
+            _reactiveEnginesRemove;
 
-                        using (var profiler = new PlatformProfiler("Svelto.ECS - Entities Submission"))
-                        {
-                            var iterations       = 0;
-                            var hasEverSubmitted = false;
-#if UNITY_NATIVE
-                            enginesRootTarget.FlushNativeOperations(profiler);
-#endif
-                            //todo: proper unit test structural changes made as result of add/remove callbacks
-                            while (enginesRootTarget.HasMadeNewStructuralChangesInThisIteration() && iterations++ < 5)
-                            {
-                                hasEverSubmitted = true;
-
-                                while (true)
-                                {
-                                    _privateSubmitEntities.MoveNext();
-                                    if (_privateSubmitEntities.Current == true)
-                                    {
-                                        using (profiler.Yield())
-                                        {
-                                            yield return true;
-                                        }
-                                    }
-                                    else
-                                        break;
-                                }
-#if UNITY_NATIVE
-                                if (enginesRootTarget.HasMadeNewStructuralChangesInThisIteration())
-                                    enginesRootTarget.FlushNativeOperations(profiler);
-#endif
-                            }
+        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnRemoveEx>>>
+            _reactiveEnginesRemoveEx;
 
-#if DEBUG && !PROFILE_SVELTO
-                            if (iterations == 5)
-                                throw new ECSException("possible circular submission detected");
-#endif
-                            if (hasEverSubmitted)
-                                enginesRootTarget.NotifyReactiveEnginesOnSubmission();
-                        }
+        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnSwap>>> _reactiveEnginesSwap;
 
-                        entitiesSubmissionScheduler.isRunning = false;
-                        ++entitiesSubmissionScheduler.iteration;
-                    }
+        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnSwapEx>>>
+            _reactiveEnginesSwapEx;
 
-                    yield return false;
-                }
-            }
+        readonly FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnDispose>>>
+            _reactiveEnginesDispose;
 
-            public uint maxNumberOfOperationsPerFrame
-            {
-                set => _enginesRoot.Target._maxNumberOfOperationsPerFrame = value;
-            }
-
-            readonly Svelto.DataStructures.WeakReference<EnginesRoot> _enginesRoot;
+        readonly FasterList<IReactOnSubmission> _reactiveEnginesSubmission;
+    }
 
-            internal readonly IEnumerator<bool> submitEntities;
-            readonly          IEnumerator<bool> _privateSubmitEntities;
-        }
+    public enum EnginesReadyOption
+    {
+        ReadyAsAdded,
+        WaitForReady
     }
 }

Core/EnginesRoot.Entities.cs

@@ -1,7 +1,8 @@
-using System;
+//#define PARANOID_CHECK
+
+using System;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
-using Svelto.Common;
 using Svelto.DataStructures;
 using Svelto.ECS.Internal;
 
@@ -16,27 +17,34 @@ namespace Svelto.ECS
             return new GenericEntityStreamConsumerFactory(this);
         }
 
-        public IEntityFactory GenerateEntityFactory() { return new GenericEntityFactory(this); }
-        public IEntityFunctions GenerateEntityFunctions() { return new GenericEntityFunctions(this); }
+        public IEntityFactory GenerateEntityFactory()
+        {
+            return new GenericEntityFactory(this);
+        }
+
+        public IEntityFunctions GenerateEntityFunctions()
+        {
+            return new GenericEntityFunctions(this);
+        }
 
         ///--------------------------------------------
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        EntityInitializer BuildEntity(EGID entityID, IComponentBuilder[] componentsToBuild, Type descriptorType
-                                    , IEnumerable<object> implementors = null)
+        EntityInitializer BuildEntity(EGID entityID, IComponentBuilder[] componentsToBuild, Type descriptorType,
+            IEnumerable<object> implementors, string caller)
         {
-            CheckAddEntityID(entityID, descriptorType);
+            CheckAddEntityID(entityID, descriptorType, caller);
 
-            DBC.ECS.Check.Require(entityID.groupID.isInvalid == false
-                        , "invalid group detected, are you using new ExclusiveGroupStruct() instead of new ExclusiveGroup()?");
+            DBC.ECS.Check.Require(entityID.groupID.isInvalid == false,
+                "invalid group detected, are you using new ExclusiveGroupStruct() instead of new ExclusiveGroup()?");
 
             var reference = _entityLocator.ClaimReference();
             _entityLocator.SetReference(reference, entityID);
 
             var dic = EntityFactory.BuildGroupedEntities(entityID, _groupedEntityToAdd, componentsToBuild, implementors
 #if DEBUG && !PROFILE_SVELTO
-                                                       , descriptorType
+              , descriptorType
 #endif
-                                                         );
+            );
 
             return new EntityInitializer(entityID, dic, reference);
         }
@@ -44,17 +52,17 @@ namespace Svelto.ECS
         /// <summary>
         /// Preallocate memory to avoid the impact to resize arrays when many entities are submitted at once
         /// </summary>
-        void Preallocate(ExclusiveGroupStruct groupID, uint numberOfEntities, IComponentBuilder[] entityComponentsToBuild)
+        void Preallocate(ExclusiveGroupStruct groupID, uint size, IComponentBuilder[] entityComponentsToBuild)
         {
             void PreallocateEntitiesToAdd()
             {
-                _groupedEntityToAdd.Preallocate(groupID, numberOfEntities, entityComponentsToBuild);
+                _groupedEntityToAdd.Preallocate(groupID, size, entityComponentsToBuild);
             }
 
             void PreallocateDBGroup()
             {
                 var numberOfEntityComponents = entityComponentsToBuild.Length;
-                FasterDictionary<RefWrapperType, ITypeSafeDictionary> group = GetOrCreateDBGroup(groupID);
+                FasterDictionary<RefWrapperType, ITypeSafeDictionary> group = GetOrAddDBGroup(groupID);
 
                 for (var index = 0; index < numberOfEntityComponents; index++)
                 {
@@ -62,8 +70,8 @@ namespace Svelto.ECS
                     var entityComponentType    = entityComponentBuilder.GetEntityComponentType();
 
                     var refWrapper = new RefWrapperType(entityComponentType);
-                    var dbList         = group.GetOrCreate(refWrapper, ()=>entityComponentBuilder.CreateDictionary(numberOfEntities));
-                    entityComponentBuilder.Preallocate(dbList, numberOfEntities);
+                    var dbList     = group.GetOrAdd(refWrapper, () => entityComponentBuilder.CreateDictionary(size));
+                    entityComponentBuilder.Preallocate(dbList, size);
 
                     if (_groupsPerEntity.TryGetValue(refWrapper, out var groupedGroup) == false)
                         groupedGroup = _groupsPerEntity[refWrapper] =
@@ -75,245 +83,82 @@ namespace Svelto.ECS
 
             PreallocateDBGroup();
             PreallocateEntitiesToAdd();
-            _entityLocator.PreallocateReferenceMaps(groupID, numberOfEntities);
-        }
-
-        ///--------------------------------------------
-        ///
-        void MoveEntityFromAndToEngines(IComponentBuilder[] componentBuilders, EGID fromEntityGID, EGID? toEntityGID)
-        {
-            using (var sampler = new PlatformProfiler("Move Entity From Engines"))
-            {
-                var fromGroup = GetDBGroup(fromEntityGID.groupID);
-
-                //Check if there is an EntityInfo linked to this entity, if so it's a DynamicEntityDescriptor!
-                if (fromGroup.TryGetValue(new RefWrapperType(ComponentBuilderUtilities.ENTITY_INFO_COMPONENT)
-                                        , out var entityInfoDic)
-                 && ((ITypeSafeDictionary<EntityInfoComponent>) entityInfoDic).TryGetValue(
-                        fromEntityGID.entityID, out var entityInfo))
-                    SwapOrRemoveEntityComponents(fromEntityGID, toEntityGID, entityInfo.componentsToBuild, fromGroup
-                                               , sampler);
-                //otherwise it's a normal static entity descriptor
-                else
-                    SwapOrRemoveEntityComponents(fromEntityGID, toEntityGID, componentBuilders, fromGroup, sampler);
-            }
-        }
-
-        void SwapOrRemoveEntityComponents
-        (EGID fromEntityGID, EGID? toEntityGID, IComponentBuilder[] entitiesToMove
-       , FasterDictionary<RefWrapperType, ITypeSafeDictionary> fromGroup, in PlatformProfiler sampler)
-        {
-            using (sampler.Sample("MoveEntityComponents"))
-            {
-                var length = entitiesToMove.Length;
-
-                FasterDictionary<RefWrapperType, ITypeSafeDictionary> toGroup = null;
-
-                //Swap is not like adding a new entity. While adding new entities happen at the end of submission
-                //Adding an entity to a group due to a swap of groups happens now.
-                if (toEntityGID.HasValue)
-                {
-                    var entityGid = toEntityGID.Value;
-                    _entityLocator.UpdateEntityReference(fromEntityGID, entityGid);
-
-                    var toGroupID = entityGid.groupID;
-
-                    toGroup = GetOrCreateDBGroup(toGroupID);
-
-                    //Add all the entities to the dictionary
-                    for (var i = 0; i < length; i++)
-                        CopyEntityToDictionary(fromEntityGID, entityGid, fromGroup, toGroup
-                                             , entitiesToMove[i].GetEntityComponentType(), sampler);
-                }
-                else
-                {
-                    _entityLocator.RemoveEntityReference(fromEntityGID);
-                }
-
-                //call all the callbacks
-                for (var i = 0; i < length; i++)
-                    ExecuteEnginesSwapOrRemoveCallbacks(fromEntityGID, toEntityGID, fromGroup, toGroup
-                                                      , entitiesToMove[i].GetEntityComponentType(), sampler);
-
-                //then remove all the entities from the dictionary
-                for (var i = 0; i < length; i++)
-                    RemoveEntityFromDictionary(fromEntityGID, fromGroup, entitiesToMove[i].GetEntityComponentType()
-                                             , sampler);
-            }
-        }
-
-        void CopyEntityToDictionary
-        (EGID entityGID, EGID toEntityGID, FasterDictionary<RefWrapperType, ITypeSafeDictionary> fromGroup
-       , FasterDictionary<RefWrapperType, ITypeSafeDictionary> toGroup, Type entityComponentType
-       , in PlatformProfiler sampler)
-        {
-            using (sampler.Sample("CopyEntityToDictionary"))
-            {
-                var wrapper = new RefWrapperType(entityComponentType);
-
-                ITypeSafeDictionary fromTypeSafeDictionary =
-                    GetTypeSafeDictionary(entityGID.groupID, fromGroup, wrapper);
-
-#if DEBUG && !PROFILE_SVELTO
-                if (fromTypeSafeDictionary.Has(entityGID.entityID) == false)
-                {
-                    throw new EntityNotFoundException(entityGID, entityComponentType);
-                }
-#endif
-                ITypeSafeDictionary toEntitiesDictionary =
-                    GetOrCreateTypeSafeDictionary(toEntityGID.groupID, toGroup, wrapper, fromTypeSafeDictionary);
-
-                fromTypeSafeDictionary.AddEntityToDictionary(entityGID, toEntityGID, toEntitiesDictionary);
-            }
-        }
-
-        void ExecuteEnginesSwapOrRemoveCallbacks
-        (EGID entityGID, EGID? toEntityGID, FasterDictionary<RefWrapperType, ITypeSafeDictionary> fromGroup
-       , FasterDictionary<RefWrapperType, ITypeSafeDictionary> toGroup, Type entityComponentType
-       , in PlatformProfiler profiler)
-        {
-            using (profiler.Sample("MoveEntityComponentFromAndToEngines"))
-            {
-                //add all the entities
-                var refWrapper             = new RefWrapperType(entityComponentType);
-                var fromTypeSafeDictionary = GetTypeSafeDictionary(entityGID.groupID, fromGroup, refWrapper);
-
-                ITypeSafeDictionary toEntitiesDictionary = null;
-                if (toGroup != null)
-                    toEntitiesDictionary = toGroup[refWrapper]; //this is guaranteed to exist by AddEntityToDictionary
-
-#if DEBUG && !PROFILE_SVELTO
-                if (fromTypeSafeDictionary.Has(entityGID.entityID) == false)
-                    throw new EntityNotFoundException(entityGID, entityComponentType);
-#endif
-                fromTypeSafeDictionary.ExecuteEnginesSwapOrRemoveCallbacks(entityGID, toEntityGID, toEntitiesDictionary
-                                                                         , toEntityGID == null
-                                                                               ? _reactiveEnginesAddRemove
-                                                                               : _reactiveEnginesSwap, in profiler);
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        void RemoveEntityFromDictionary
-        (EGID entityGID, FasterDictionary<RefWrapperType, ITypeSafeDictionary> fromGroup, Type entityComponentType
-       , in PlatformProfiler sampler)
-        {
-            using (sampler.Sample("RemoveEntityFromDictionary"))
-            {
-                var refWrapper             = new RefWrapperType(entityComponentType);
-                var fromTypeSafeDictionary = GetTypeSafeDictionary(entityGID.groupID, fromGroup, refWrapper);
-
-                fromTypeSafeDictionary.RemoveEntityFromDictionary(entityGID);
-            }
-        }
-
-        /// <summary>
-        /// Swap all the entities from one group to another
-        ///
-        /// TODO: write unit test that also tests that this calls MoveTo callbacks and not Add or Remove.
-        /// also that the passing EGID is the same of a component with EGID
-        /// </summary>
-        /// <param name="fromIdGroupId"></param>
-        /// <param name="toGroupId"></param>
-        /// <param name="profiler"></param>
-        void SwapEntitiesBetweenGroups
-            (ExclusiveGroupStruct fromIdGroupId, ExclusiveGroupStruct toGroupId, in PlatformProfiler profiler)
-        {
-            using (profiler.Sample("SwapEntitiesBetweenGroups"))
-            {
-                FasterDictionary<RefWrapperType, ITypeSafeDictionary> fromGroup = GetDBGroup(fromIdGroupId);
-                FasterDictionary<RefWrapperType, ITypeSafeDictionary> toGroup   = GetOrCreateDBGroup(toGroupId);
-
-                _entityLocator.UpdateAllGroupReferenceLocators(fromIdGroupId, toGroupId);
-
-                foreach (var dictionaryOfEntities in fromGroup)
-                {
-                    ITypeSafeDictionary toEntitiesDictionary =
-                        GetOrCreateTypeSafeDictionary(toGroupId, toGroup, dictionaryOfEntities.Key
-                                                    , dictionaryOfEntities.Value);
-
-                    var groupsOfEntityType = _groupsPerEntity[dictionaryOfEntities.Key];
-                    var groupOfEntitiesToCopyAndClear = groupsOfEntityType[fromIdGroupId];
-
-                    toEntitiesDictionary.AddEntitiesFromDictionary(groupOfEntitiesToCopyAndClear, toGroupId, this);
-
-                    //call all the MoveTo callbacks
-                    dictionaryOfEntities.Value.ExecuteEnginesAddOrSwapCallbacks(_reactiveEnginesSwap
-                      , dictionaryOfEntities.Value, fromIdGroupId, toGroupId, profiler);
-
-                    //todo: if it's unmanaged, I can use fastclear
-                    groupOfEntitiesToCopyAndClear.Clear();
-                }
-            }
+            _entityLocator.PreallocateReferenceMaps(groupID, size);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         FasterDictionary<RefWrapperType, ITypeSafeDictionary> GetDBGroup(ExclusiveGroupStruct fromIdGroupId)
         {
-            if (_groupEntityComponentsDB.TryGetValue(fromIdGroupId
-                                                   , out FasterDictionary<RefWrapperType, ITypeSafeDictionary>
-                                                         fromGroup) == false)
+            if (_groupEntityComponentsDB.TryGetValue(fromIdGroupId,
+                    out FasterDictionary<RefWrapperType, ITypeSafeDictionary> fromGroup) == false)
                 throw new ECSException("Group doesn't exist: ".FastConcat(fromIdGroupId.ToName()));
 
             return fromGroup;
         }
 
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        FasterDictionary<RefWrapperType, ITypeSafeDictionary> GetOrCreateDBGroup
-            (ExclusiveGroupStruct toGroupId)
+        FasterDictionary<RefWrapperType, ITypeSafeDictionary> GetOrAddDBGroup(ExclusiveGroupStruct toGroupId)
         {
-            return _groupEntityComponentsDB.GetOrCreate(
-                toGroupId, () => new FasterDictionary<RefWrapperType, ITypeSafeDictionary>());
+            return _groupEntityComponentsDB.GetOrAdd(toGroupId,
+                () => new FasterDictionary<RefWrapperType, ITypeSafeDictionary>());
         }
 
-        ITypeSafeDictionary GetOrCreateTypeSafeDictionary
-        (ExclusiveGroupStruct groupId, FasterDictionary<RefWrapperType, ITypeSafeDictionary> toGroup
-       , RefWrapperType type, ITypeSafeDictionary fromTypeSafeDictionary)
+        IComponentBuilder[] FindRealComponents<T>(EGID fromEntityGID) where T : IEntityDescriptor, new()
         {
-            //be sure that the TypeSafeDictionary for the entity Type exists
-            if (toGroup.TryGetValue(type, out ITypeSafeDictionary toEntitiesDictionary) == false)
-            {
-                toEntitiesDictionary = fromTypeSafeDictionary.Create();
-                toGroup.Add(type, toEntitiesDictionary);
-            }
+            var fromGroup = GetDBGroup(fromEntityGID.groupID);
 
-            //update GroupsPerEntity
-            if (_groupsPerEntity.TryGetValue(type, out var groupedGroup) == false)
-                groupedGroup = _groupsPerEntity[type] =
-                    new FasterDictionary<ExclusiveGroupStruct, ITypeSafeDictionary>();
+            if (fromGroup.TryGetValue(new RefWrapperType(ComponentBuilderUtilities.ENTITY_INFO_COMPONENT),
+                    out var entityInfoDic) //<entity ID, EntityInfoComponent>
+             && ((ITypeSafeDictionary<EntityInfoComponent>)entityInfoDic).TryGetValue(fromEntityGID.entityID,
+                    out var entityInfo)) //there could be multiple entity descriptors registered in the same group, so it's necessary to check if the entity registered in the group has entityInfoComponent   
+            {
+#if PARANOID_CHECK
+                var hash = new HashSet<IComponentBuilder>(entityInfo.componentsToBuild,
+                    default(ComponentBuilderComparer));
 
-            groupedGroup[groupId] = toEntitiesDictionary;
-            return toEntitiesDictionary;
-        }
+                foreach (var component in EntityDescriptorTemplate<T>.descriptor.componentsToBuild)
+                {
+                    if (hash.Contains(component) == false)
+                        throw new Exception(
+                            $"entityInfo.componentsToBuild must contain all the base components {fromEntityGID}," +
+                            $" missing component {component}");
 
-        static ITypeSafeDictionary GetTypeSafeDictionary
-            (ExclusiveGroupStruct groupID, FasterDictionary<RefWrapperType, ITypeSafeDictionary> @group, RefWrapperType refWrapper)
-        {
-            if (@group.TryGetValue(refWrapper, out ITypeSafeDictionary fromTypeSafeDictionary) == false)
-            {
-                throw new ECSException("no group found: ".FastConcat(groupID.ToName()));
+                    hash.Remove(component);
+                }
+#endif
+                return entityInfo.componentsToBuild;
             }
 
-            return fromTypeSafeDictionary;
+            return EntityDescriptorTemplate<T>.descriptor.componentsToBuild;
         }
 
-        void RemoveEntitiesFromGroup(ExclusiveGroupStruct groupID, in PlatformProfiler profiler)
+        IComponentBuilder[] FindRealComponents(EGID fromEntityGID, IComponentBuilder[] baseComponents)
         {
-            _entityLocator.RemoveAllGroupReferenceLocators(groupID);
+            var fromGroup = GetDBGroup(fromEntityGID.groupID);
 
-            if (_groupEntityComponentsDB.TryGetValue(groupID, out var dictionariesOfEntities))
+            if (fromGroup.TryGetValue(new RefWrapperType(ComponentBuilderUtilities.ENTITY_INFO_COMPONENT),
+                    out var entityInfoDic) //<entity ID, EntityInfoComponent>
+             && ((ITypeSafeDictionary<EntityInfoComponent>)entityInfoDic).TryGetValue(fromEntityGID.entityID,
+                    out var entityInfo)) //there could be multiple entity descriptors registered in the same group, so it's necessary to check if the entity registered in the group has entityInfoComponent   
             {
-                foreach (var dictionaryOfEntities in dictionariesOfEntities)
+#if PARANOID_CHECK
+                var hash = new HashSet<IComponentBuilder>(entityInfo.componentsToBuild,
+                    default(ComponentBuilderComparer));
+
+                foreach (var component in baseComponents)
                 {
-                    dictionaryOfEntities.Value.ExecuteEnginesRemoveCallbacks(_reactiveEnginesAddRemove, profiler
-                                                                           , groupID);
-                    dictionaryOfEntities.Value.FastClear();
+                    if (hash.Contains(component) == false)
+                        throw new Exception(
+                            $"entityInfo.componentsToBuild must contain all the base components {fromEntityGID}," +
+                            $" missing component {component}");
 
-                    var groupsOfEntityType = _groupsPerEntity[dictionaryOfEntities.Key];
-                    groupsOfEntityType[groupID].FastClear();
+                    hash.Remove(component);
                 }
+#endif
+                return entityInfo.componentsToBuild;
             }
+
+            return baseComponents;
         }
 
         //one datastructure rule them all:
@@ -322,7 +167,7 @@ namespace Svelto.ECS
         //to the FasterDictionary capabilities OR it's possible to get a specific entityComponent indexed by
         //ID. This ID doesn't need to be the EGID, it can be just the entityID
         //for each group id, save a dictionary indexed by entity type of entities indexed by id
-        //                         group                EntityComponentType   entityID, EntityComponent
+        //                                        group                  EntityComponentType     entityID, EntityComponent
         internal readonly FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, ITypeSafeDictionary>>
             _groupEntityComponentsDB;
 
@@ -334,7 +179,7 @@ namespace Svelto.ECS
             _groupsPerEntity;
 
         //The filters stored for each component and group
-        internal readonly FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, GroupFilters>>
+        internal readonly FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, LegacyGroupFilters>>
             _groupFilters;
 
         readonly EntitiesDB _entitiesDB;

Core/EnginesRoot.GenericEntityFactory.cs

@@ -1,8 +1,9 @@
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using Svelto.Common;
 
- namespace Svelto.ECS
+namespace Svelto.ECS
 {
     public partial class EnginesRoot
     {
@@ -14,52 +15,61 @@ using Svelto.Common;
             }
 
             public EntityInitializer BuildEntity<T>
-                (uint entityID, ExclusiveBuildGroup groupStructId, IEnumerable<object> implementors = null)
-                where T : IEntityDescriptor, new()
+            (uint entityID, ExclusiveBuildGroup groupStructId, IEnumerable<object> implementors = null
+           , [CallerMemberName] string caller = null) where T : IEntityDescriptor, new()
             {
                 return _enginesRoot.Target.BuildEntity(new EGID(entityID, groupStructId)
                                                      , EntityDescriptorTemplate<T>.descriptor.componentsToBuild
-                                                     , TypeCache<T>.type, implementors);
+                                                     , TypeCache<T>.type, implementors, caller);
             }
 
-            public EntityInitializer BuildEntity<T>(EGID egid, IEnumerable<object> implementors = null)
-                where T : IEntityDescriptor, new()
+            public EntityInitializer BuildEntity<T>
+            (EGID egid, IEnumerable<object> implementors = null
+           , [CallerMemberName] string caller = null) where T : IEntityDescriptor, new()
             {
-                return _enginesRoot.Target.BuildEntity(
-                    egid, EntityDescriptorTemplate<T>.descriptor.componentsToBuild, TypeCache<T>.type, implementors);
+                return _enginesRoot.Target.BuildEntity(egid, EntityDescriptorTemplate<T>.descriptor.componentsToBuild
+                                                     , TypeCache<T>.type, implementors, caller);
             }
 
             public EntityInitializer BuildEntity<T>
-                (EGID egid, T entityDescriptor, IEnumerable<object> implementors) where T : IEntityDescriptor
+            (EGID egid, T entityDescriptor, IEnumerable<object> implementors
+           , [CallerMemberName] string caller = null) where T : IEntityDescriptor
             {
-                return _enginesRoot.Target.BuildEntity(egid, entityDescriptor.componentsToBuild, TypeCache<T>.type, implementors);
+                return _enginesRoot.Target.BuildEntity(egid, entityDescriptor.componentsToBuild, TypeCache<T>.type
+                                                     , implementors, caller);
             }
 
             public EntityInitializer BuildEntity<T>
-                (uint entityID, ExclusiveBuildGroup groupStructId, T descriptorEntity, IEnumerable<object> implementors)
-                where T : IEntityDescriptor
+            (uint entityID, ExclusiveBuildGroup groupStructId, T descriptorEntity, IEnumerable<object> implementors
+           , [CallerMemberName] string caller = null) where T : IEntityDescriptor
             {
                 return _enginesRoot.Target.BuildEntity(new EGID(entityID, groupStructId)
-                                                     , descriptorEntity.componentsToBuild, TypeCache<T>.type, implementors);
+                                                     , descriptorEntity.componentsToBuild, TypeCache<T>.type
+                                                     , implementors, caller);
             }
 
             public void PreallocateEntitySpace<T>(ExclusiveGroupStruct groupStructId, uint numberOfEntities)
                 where T : IEntityDescriptor, new()
             {
-                _enginesRoot.Target.Preallocate(groupStructId, numberOfEntities, EntityDescriptorTemplate<T>.descriptor.componentsToBuild);
+                _enginesRoot.Target.Preallocate(groupStructId, numberOfEntities
+                                              , EntityDescriptorTemplate<T>.descriptor.componentsToBuild);
             }
-            
-            public EntityInitializer BuildEntity(EGID egid, IComponentBuilder[] componentsToBuild, Type type, IEnumerable<object> implementors = null)
+
+            public EntityInitializer BuildEntity
+            (EGID egid, IComponentBuilder[] componentsToBuild, Type type, IEnumerable<object> implementors = null
+           , [CallerMemberName] string caller = null)
             {
-                return _enginesRoot.Target.BuildEntity(egid, componentsToBuild, type, implementors);
+                return _enginesRoot.Target.BuildEntity(egid, componentsToBuild, type, implementors, caller);
             }
-            
+
 #if UNITY_NATIVE
-            public Svelto.ECS.Native.NativeEntityFactory ToNative<T>(string callerName) where T : IEntityDescriptor, new()
+            public Native.NativeEntityFactory ToNative<T>
+                ([CallerMemberName] string caller = null)
+                where T : IEntityDescriptor, new()
             {
-                return _enginesRoot.Target.ProvideNativeEntityFactoryQueue<T>(callerName);
+                return _enginesRoot.Target.ProvideNativeEntityFactoryQueue<T>(caller);
             }
-#endif            
+#endif
 
             //enginesRoot is a weakreference because GenericEntityStreamConsumerFactory can be injected inside
             //engines of other enginesRoot

Core/EnginesRoot.GenericEntityFunctions.cs

@@ -7,193 +7,146 @@ namespace Svelto.ECS
 {
     public partial class EnginesRoot
     {
-        /// <summary>
-        /// todo: EnginesRoot was a weakreference to give the change to inject
-        /// entity functions from other engines root. It probably should be reverted
-        /// </summary>
         class GenericEntityFunctions : IEntityFunctions
         {
             internal GenericEntityFunctions(EnginesRoot weakReference)
             {
-                _enginesRoot = new Svelto.DataStructures.WeakReference<EnginesRoot>(weakReference);
+                _enginesRoot = new WeakReference<EnginesRoot>(weakReference);
             }
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void RemoveEntity<T>(uint entityID, ExclusiveBuildGroup groupID) where T :
-                IEntityDescriptor, new()
+            public void RemoveEntity<T>
+                (uint entityID, ExclusiveBuildGroup groupID, [CallerMemberName] string caller = null)
+                where T : IEntityDescriptor, new()
             {
-                RemoveEntity<T>(new EGID(entityID, groupID));
+                RemoveEntity<T>(new EGID(entityID, groupID), caller);
             }
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void RemoveEntity<T>(EGID entityEGID) where T : IEntityDescriptor, new()
+            public void RemoveEntity<T>(EGID entityEGID, [CallerMemberName] string caller = null) 
+                where T : IEntityDescriptor, new()
             {
                 DBC.ECS.Check.Require(entityEGID.groupID.isInvalid == false, "invalid group detected");
-                var descriptorComponentsToBuild = EntityDescriptorTemplate<T>.descriptor.componentsToBuild;
-                _enginesRoot.Target.CheckRemoveEntityID(entityEGID, TypeCache<T>.type);
+                _enginesRoot.Target.CheckRemoveEntityID(entityEGID, TypeCache<T>.type, caller);
 
-                _enginesRoot.Target.QueueEntitySubmitOperation<T>(
-                    new EntitySubmitOperation(EntitySubmitOperationType.Remove, entityEGID, entityEGID,
-                        descriptorComponentsToBuild));
+                _enginesRoot.Target.QueueRemoveEntityOperation(
+                    entityEGID, _enginesRoot.Target.FindRealComponents<T>(entityEGID), caller);
             }
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void RemoveEntitiesFromGroup(ExclusiveBuildGroup groupID)
+            public void RemoveEntitiesFromGroup(ExclusiveBuildGroup groupID, [CallerMemberName] string caller = null)
             {
                 DBC.ECS.Check.Require(groupID.isInvalid == false, "invalid group detected");
                 _enginesRoot.Target.RemoveGroupID(groupID);
 
-                _enginesRoot.Target.QueueEntitySubmitOperation(
-                    new EntitySubmitOperation(EntitySubmitOperationType.RemoveGroup, new EGID(0, groupID), new EGID()));
+                _enginesRoot.Target.QueueRemoveGroupOperation(groupID, caller);
             }
 
-            // [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            // void RemoveAllEntities<D, S>(ExclusiveGroup group)
-            //     where D : IEntityDescriptor, new() where S : unmanaged, IEntityComponent
-            // {
-            //     var targetEntitiesDB = _enginesRoot.Target._entitiesDB;
-            //     var (buffer, count) = targetEntitiesDB.QueryEntities<S>(@group);
-            //     for (uint i = 0; i < count; ++i)
-            //     {
-            //         RemoveEntity<D>(new EGID(i, group));
-            //     }
-            // }
-            //
-            // [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            // void RemoveAllEntities<D, S>()
-            //     where D : IEntityDescriptor, new() where S : unmanaged, IEntityComponent
-            // {
-            //     var  targetEntitiesDB = _enginesRoot.Target._entitiesDB;
-            //     foreach (var ((buffer, count), exclusiveGroupStruct) in targetEntitiesDB.QueryEntities<S>())
-            //         for (uint i = 0; i < count; ++i)
-            //         {
-            //             RemoveEntity<D>(new EGID(i, exclusiveGroupStruct));
-            //         }
-            // }
-
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void SwapEntitiesInGroup<T>(ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID)
-                where T : IEntityDescriptor, new()
+            public void SwapEntitiesInGroup
+            (ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID, [CallerMemberName] string caller = null) 
             {
                 if (_enginesRoot.Target._groupEntityComponentsDB.TryGetValue(
-                        fromGroupID.group, out FasterDictionary<RefWrapperType, ITypeSafeDictionary> entitiesInGroupPerType)
-                 == true)
+                        fromGroupID.group
+                      , out FasterDictionary<RefWrapperType, ITypeSafeDictionary> entitiesInGroupPerType) == true)
                 {
 #if DEBUG && !PROFILE_SVELTO
-                    IComponentBuilder[] components = EntityDescriptorTemplate<T>.descriptor.componentsToBuild;
-                    var dictionary = entitiesInGroupPerType[new RefWrapperType(components[0].GetEntityComponentType())];
+                    ITypeSafeDictionary dictionary = entitiesInGroupPerType.unsafeValues[0];
 
                     dictionary.KeysEvaluator((key) =>
                     {
-                        _enginesRoot.Target.CheckRemoveEntityID(new EGID(key, fromGroupID), TypeCache<T>.type);
-                        _enginesRoot.Target.CheckAddEntityID(new EGID(key, toGroupID), TypeCache<T>.type);
+                        _enginesRoot.Target.CheckRemoveEntityID(new EGID(key, fromGroupID), null, caller);
+                        _enginesRoot.Target.CheckAddEntityID(new EGID(key, toGroupID), null, caller);
                     });
-
 #endif
-                    _enginesRoot.Target.QueueEntitySubmitOperation(
-                        new EntitySubmitOperation(EntitySubmitOperationType.SwapGroup, new EGID(0, fromGroupID)
-                                                , new EGID(0, toGroupID)));
+                    _enginesRoot.Target.QueueSwapGroupOperation(fromGroupID, toGroupID, caller);
                 }
             }
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void SwapEntityGroup<T>(uint entityID, ExclusiveBuildGroup fromGroupID,
-                                           ExclusiveBuildGroup toGroupID)
-                where T : IEntityDescriptor, new()
+            public void SwapEntityGroup<T>
+            (uint entityID, ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID
+           , [CallerMemberName] string caller = null) where T : IEntityDescriptor, new()
             {
-                SwapEntityGroup<T>(new EGID(entityID, fromGroupID), toGroupID);
+                SwapEntityGroup<T>(new EGID(entityID, fromGroupID), toGroupID, caller);
             }
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void SwapEntityGroup<T>(EGID fromID, ExclusiveBuildGroup toGroupID)
+            public void SwapEntityGroup<T>
+                (EGID fromEGID, ExclusiveBuildGroup toGroupID, [CallerMemberName] string caller = null)
                 where T : IEntityDescriptor, new()
             {
-                SwapEntityGroup<T>(fromID, new EGID(fromID.entityID, toGroupID));
+                SwapEntityGroup<T>(fromEGID, new EGID(fromEGID.entityID, toGroupID), caller);
             }
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void SwapEntityGroup<T>(EGID fromID, ExclusiveBuildGroup mustBeFromGroup, ExclusiveBuildGroup toGroupID)
+            public void SwapEntityGroup<T>
+                (EGID fromEGID, EGID toEGID, ExclusiveBuildGroup mustBeFromGroup, [CallerMemberName] string caller = null)
                 where T : IEntityDescriptor, new()
             {
-                if (fromID.groupID != mustBeFromGroup)
-                    throw new ECSException($"Entity is not coming from the expected group. Expected {mustBeFromGroup} is {fromID.groupID}");
+                if (fromEGID.groupID != mustBeFromGroup)
+                    throw new ECSException(
+                        $"Entity is not coming from the expected group Expected {mustBeFromGroup} is {fromEGID.groupID}");
 
-                SwapEntityGroup<T>(fromID, toGroupID);
+                SwapEntityGroup<T>(fromEGID, toEGID, caller);
             }
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void SwapEntityGroup<T>(EGID fromID, EGID toID, ExclusiveBuildGroup mustBeFromGroup)
+            public void SwapEntityGroup<T>(EGID fromEGID, EGID toEGID, [CallerMemberName] string caller = null)
                 where T : IEntityDescriptor, new()
             {
-                if (fromID.groupID != mustBeFromGroup)
-                    throw new ECSException($"Entity is not coming from the expected group Expected {mustBeFromGroup} is {fromID.groupID}");
+                DBC.ECS.Check.Require(fromEGID.groupID.isInvalid == false, "invalid group detected");
+                DBC.ECS.Check.Require(toEGID.groupID.isInvalid == false, "invalid group detected");
+
+                var enginesRootTarget = _enginesRoot.Target;
 
-                SwapEntityGroup<T>(fromID, toID);
+                enginesRootTarget.CheckRemoveEntityID(fromEGID, TypeCache<T>.type, caller);
+                enginesRootTarget.CheckAddEntityID(toEGID, TypeCache<T>.type, caller);
+
+                enginesRootTarget.QueueSwapEntityOperation(fromEGID, toEGID
+                                                         , this._enginesRoot.Target.FindRealComponents<T>(fromEGID)
+                                                         , caller);
             }
 
 #if UNITY_NATIVE
-            public Svelto.ECS.Native.NativeEntityRemove ToNativeRemove<T>(string memberName) where T : IEntityDescriptor, new()
+            public Native.NativeEntityRemove ToNativeRemove<T>(string memberName) where T : IEntityDescriptor, new()
             {
                 return _enginesRoot.Target.ProvideNativeEntityRemoveQueue<T>(memberName);
             }
 
-            public Svelto.ECS.Native.NativeEntitySwap ToNativeSwap<T>(string memberName) where T : IEntityDescriptor, new()
+            public Native.NativeEntitySwap ToNativeSwap<T>(string memberName) where T : IEntityDescriptor, new()
             {
                 return _enginesRoot.Target.ProvideNativeEntitySwapQueue<T>(memberName);
             }
 #endif
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void SwapEntityGroup<T>(EGID fromID, EGID toID)
-                where T : IEntityDescriptor, new()
-            {
-                DBC.ECS.Check.Require(fromID.groupID.isInvalid == false, "invalid group detected");
-                DBC.ECS.Check.Require(toID.groupID.isInvalid == false, "invalid group detected");
-
-                var enginesRootTarget           = _enginesRoot.Target;
-                var descriptorComponentsToBuild = EntityDescriptorTemplate<T>.descriptor.componentsToBuild;
-                
-                enginesRootTarget.CheckRemoveEntityID(fromID, TypeCache<T>.type);
-                enginesRootTarget.CheckAddEntityID(toID, TypeCache<T>.type);
-
-                enginesRootTarget.QueueEntitySubmitOperation<T>(
-                    new EntitySubmitOperation(EntitySubmitOperationType.Swap,
-                        fromID, toID, descriptorComponentsToBuild));
-            }
-
             //enginesRoot is a weakreference because GenericEntityStreamConsumerFactory can be injected inside
             //engines of other enginesRoot
-            readonly Svelto.DataStructures.WeakReference<EnginesRoot> _enginesRoot;
+            readonly WeakReference<EnginesRoot> _enginesRoot;
         }
 
-        void QueueEntitySubmitOperation(EntitySubmitOperation entitySubmitOperation)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        void QueueRemoveGroupOperation(ExclusiveBuildGroup groupID, string caller)
         {
-#if DEBUG && !PROFILE_SVELTO
-            entitySubmitOperation.trace = new System.Diagnostics.StackFrame(1, true);
-#endif
-            _entitiesOperations.Add((ulong) entitySubmitOperation.fromID, entitySubmitOperation);
+            _entitiesOperations.AddRemoveGroupOperation(groupID, caller);
         }
 
-        void QueueEntitySubmitOperation<T>(EntitySubmitOperation entitySubmitOperation) where T : IEntityDescriptor
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        void QueueSwapGroupOperation(ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID, string caller)
         {
-#if DEBUG && !PROFILE_SVELTO
-            entitySubmitOperation.trace = new System.Diagnostics.StackFrame(1, true);
+            _entitiesOperations.AddSwapGroupOperation(fromGroupID, toGroupID, caller);
+        }
 
-            if (_entitiesOperations.TryGetValue((ulong) entitySubmitOperation.fromID, out var entitySubmitedOperation))
-            {
-                if (entitySubmitedOperation != entitySubmitOperation)
-                    throw new ECSException("Only one entity operation per submission is allowed"
-                       .FastConcat(" entityComponentType: ")
-                       .FastConcat(typeof(T).Name)
-                       .FastConcat(" submission type ", entitySubmitOperation.type.ToString(),
-                            " from ID: ", entitySubmitOperation.fromID.entityID.ToString())
-                       .FastConcat(" previous operation type: ",
-                            _entitiesOperations[(ulong) entitySubmitOperation.fromID].type
-                               .ToString()));
-            }
-            else
-#endif
-                _entitiesOperations[(ulong) entitySubmitOperation.fromID] = entitySubmitOperation;
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        void QueueSwapEntityOperation
+            (EGID fromID, EGID toID, IComponentBuilder[] componentBuilders, string caller)
+        {
+            _entitiesOperations.AddSwapOperation(fromID, toID, componentBuilders, caller);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        void QueueRemoveEntityOperation(EGID entityEGID, IComponentBuilder[] componentBuilders, string caller)
+        {
+            _entitiesOperations.AddRemoveOperation(entityEGID, componentBuilders, caller);
         }
     }
 }

Core/EnginesRoot.Submission.cs

@@ -1,177 +1,515 @@
-using System.Collections.Generic;
+using System;
+using System.Runtime.CompilerServices;
 using Svelto.Common;
 using Svelto.DataStructures;
+using Svelto.ECS.DataStructures;
+using Svelto.ECS.Internal;
 
 namespace Svelto.ECS
 {
     public partial class EnginesRoot
     {
-        /// <summary>
-        /// Todo: it would be probably better to split even further the logic between submission and callbacks
-        /// Something to do when I will optimize the callbacks
-        /// </summary>
-        /// <param name="profiler"></param>
-        /// <param name="maxNumberOfOperations"></param>
-        IEnumerator<bool> SingleSubmission(PlatformProfiler profiler)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        void SingleSubmission(PlatformProfiler profiler)
         {
-            while (true)
+            ClearDebugChecks(); //this must be done first as I need the carry the last states after the submission
+
+            _entitiesOperations.ExecuteRemoveAndSwappingOperations(_swapEntities, _removeEntities, _removeGroup,
+                _swapGroup, this);
+
+            AddEntities(profiler);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        static void RemoveGroup(ExclusiveGroupStruct groupID, EnginesRoot enginesRoot)
+        {
+            using (var sampler = new PlatformProfiler("remove whole group"))
+            {
+                enginesRoot.RemoveEntitiesFromGroup(groupID, sampler);
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        static void SwapGroup(ExclusiveGroupStruct fromGroupID, ExclusiveGroupStruct toGroupID, EnginesRoot enginesRoot)
+        {
+            using (var sampler = new PlatformProfiler("swap whole group"))
+            {
+                enginesRoot.SwapEntitiesBetweenGroups(fromGroupID, toGroupID, sampler);
+            }
+        }
+
+        static void RemoveEntities(
+            FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, FasterList<(uint, string)>>>
+                removeOperations, FasterList<EGID> entitiesRemoved, EnginesRoot enginesRoot)
+        {
+            using (var sampler = new PlatformProfiler("remove Entities"))
+            {
+                using (sampler.Sample("Remove Entity References"))
+                {
+                    var count = entitiesRemoved.count;
+                    for (int i = 0; i < count; i++)
+                    {
+                        enginesRoot._entityLocator.RemoveEntityReference(entitiesRemoved[i]);
+                    }
+                }
+
+                using (sampler.Sample("Execute remove callbacks and remove entities"))
+                {
+                    foreach (var entitiesToRemove in removeOperations)
+                    {
+                        ExclusiveGroupStruct group               = entitiesToRemove.key;
+                        var                  fromGroupDictionary = enginesRoot.GetDBGroup(group);
+
+                        foreach (var groupedEntitiesToRemove in entitiesToRemove.value)
+                        {
+                            var                 componentType            = groupedEntitiesToRemove.key;
+                            ITypeSafeDictionary fromComponentsDictionary = fromGroupDictionary[componentType];
+
+                            FasterList<(uint, string)> infosToProcess = groupedEntitiesToRemove.value;
+
+                            fromComponentsDictionary.ExecuteEnginesRemoveCallbacks(infosToProcess,
+                                enginesRoot._reactiveEnginesRemove, group, in sampler);
+                        }
+                    }
+                }
+
+                using (sampler.Sample("Remove Entities"))
+                {
+                    foreach (var entitiesToRemove in removeOperations)
+                    {
+                        ExclusiveGroupStruct fromGroup           = entitiesToRemove.key;
+                        var                  fromGroupDictionary = enginesRoot.GetDBGroup(fromGroup);
+
+                        foreach (var groupedEntitiesToRemove in entitiesToRemove.value)
+                        {
+                            RefWrapperType      componentType            = groupedEntitiesToRemove.key;
+                            ITypeSafeDictionary fromComponentsDictionary = fromGroupDictionary[componentType];
+
+                            FasterList<(uint, string)> entityIDsToRemove = groupedEntitiesToRemove.value;
+
+                            enginesRoot.RemoveEntityFromPersistentFilters(entityIDsToRemove, fromGroup,
+                                    componentType, fromComponentsDictionary);
+                            
+                            fromComponentsDictionary.RemoveEntitiesFromDictionary(entityIDsToRemove);
+                            
+                            //store new count after the entities are removed, plus the number of entities removed
+                            enginesRoot._cachedRangeOfSubmittedIndices.Add(((uint, uint))(
+                                fromComponentsDictionary.count,
+                                fromComponentsDictionary.count + entityIDsToRemove.count));
+                        }
+                    }
+                }
+                
+                var rangeEnumerator = enginesRoot._cachedRangeOfSubmittedIndices.GetEnumerator();
+                using (sampler.Sample("Execute remove Callbacks Fast"))
+                {
+                    foreach (var entitiesToRemove in removeOperations)
+                    {
+                        ExclusiveGroupStruct group               = entitiesToRemove.key;
+                        var                  fromGroupDictionary = enginesRoot.GetDBGroup(group);
+
+                        foreach (var groupedEntitiesToRemove in entitiesToRemove.value)
+                        {
+                            rangeEnumerator.MoveNext();
+                            
+                            var                 componentType            = groupedEntitiesToRemove.key;
+                            ITypeSafeDictionary fromComponentsDictionary = fromGroupDictionary[componentType];
+
+                            //get all the engines linked to TValue
+                            if (!enginesRoot._reactiveEnginesRemoveEx.TryGetValue(new RefWrapperType(componentType),
+                                    out var entityComponentsEngines))
+                                continue;
+                            
+                            fromComponentsDictionary.ExecuteEnginesRemoveCallbacksFast(entityComponentsEngines,
+                                group, rangeEnumerator.Current, in sampler);
+                        }
+                    }
+                }
+            }
+        }
+
+        static void SwapEntities(
+            FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType,
+                FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>>> swapEntitiesOperations,
+            FasterList<(EGID, EGID)> entitiesIDSwaps, EnginesRoot enginesRoot)
+        {
+            using (var sampler = new PlatformProfiler("Swap entities between groups"))
             {
-                DBC.ECS.Check.Require(_maxNumberOfOperationsPerFrame > 0);
-                    
-                ClearChecks();
+                using (sampler.Sample("Update Entity References"))
+                {
+                    var count = entitiesIDSwaps.count;
+                    for (int i = 0; i < count; i++)
+                    {
+                        var (fromEntityGid, toEntityGid) = entitiesIDSwaps[i];
 
-                uint numberOfOperations   = 0;
+                        enginesRoot._entityLocator.UpdateEntityReference(fromEntityGid, toEntityGid);
+                    }
+                }
 
-                if (_entitiesOperations.count > 0)
+                using (sampler.Sample("Swap Entities"))
                 {
-                    using (var sample = profiler.Sample("Remove and Swap operations"))
+                    enginesRoot._cachedRangeOfSubmittedIndices.FastClear();
+                    //Entities to swap are organised in order to minimise the amount of dictionary lookups.
+                    //swapEntitiesOperations iterations happen in the following order:
+                    //for each fromGroup, get all the entities to swap for each component type.
+                    //then get the list of IDs for each ToGroup.
+                    //now swap the set of FromGroup -> ToGroup entities per ID.
+                    foreach (var entitiesToSwap in swapEntitiesOperations)
                     {
-                        _transientEntitiesOperations.FastClear();
-                        _entitiesOperations.CopyValuesTo(_transientEntitiesOperations);
-                        _entitiesOperations.FastClear();
-
-                        EntitySubmitOperation[] entitiesOperations =
-                            _transientEntitiesOperations.ToArrayFast(out var count);
-                        
-                        for (var i = 0; i < count; i++)
+                        ExclusiveGroupStruct fromGroup           = entitiesToSwap.key;
+                        var                  fromGroupDictionary = enginesRoot.GetDBGroup(fromGroup);
+
+                        //iterate all the fromgroups
+                        foreach (var groupedEntitiesToSwap in entitiesToSwap.value)
                         {
-                            try
+                            var                 componentType            = groupedEntitiesToSwap.key;
+                            ITypeSafeDictionary fromComponentsDictionary = fromGroupDictionary[componentType];
+
+                            //get the subset of togroups that come from from group
+                            foreach (var entitiesInfoToSwap in groupedEntitiesToSwap.value)
                             {
-                                switch (entitiesOperations[i].type)
+                                ExclusiveGroupStruct toGroup = entitiesInfoToSwap.key;
+                                ITypeSafeDictionary toComponentsDictionary =
+                                    enginesRoot.GetOrAddTypeSafeDictionary(toGroup,
+                                        enginesRoot.GetOrAddDBGroup(toGroup), componentType, fromComponentsDictionary);
+
+                                DBC.ECS.Check.Assert(toComponentsDictionary != null,
+                                    "something went wrong with the creation of dictionaries");
+
+                                //this list represents the set of entities that come from fromGroup and need
+                                //to be swapped to toGroup. Most of the times will be 1 of few units.
+                                FasterList<(uint, uint, string)> fromEntityToEntityIDs = entitiesInfoToSwap.value;
+
+                                int fromDictionaryCountBeforeSubmission = -1;
+
+                                if (enginesRoot._indicesOfPersistentFiltersUsedByThisComponent.TryGetValue(
+                                        new NativeRefWrapperType(componentType),
+                                        out NativeDynamicArrayCast<int> listOfFilters))
                                 {
-                                    case EntitySubmitOperationType.Swap:
-                                        MoveEntityFromAndToEngines(entitiesOperations[i].builders
-                                                                 , entitiesOperations[i].fromID
-                                                                 , entitiesOperations[i].toID);
-                                        break;
-                                    case EntitySubmitOperationType.Remove:
-                                        MoveEntityFromAndToEngines(entitiesOperations[i].builders
-                                                                 , entitiesOperations[i].fromID, null);
-                                        break;
-                                    case EntitySubmitOperationType.RemoveGroup:
-                                        RemoveEntitiesFromGroup(entitiesOperations[i].fromID.groupID, profiler);
-                                        break;
-                                    case EntitySubmitOperationType.SwapGroup:
-                                        SwapEntitiesBetweenGroups(entitiesOperations[i].fromID.groupID
-                                                                , entitiesOperations[i].toID.groupID, profiler);
-                                        break;
+                                    enginesRoot._cachedIndicesToSwapBeforeSubmissionForFilters.FastClear();
+
+                                    fromDictionaryCountBeforeSubmission = fromComponentsDictionary.count - 1;
+
+                                    //add the index of the entities in the component array for each entityID
+                                    //BEFORE the submission, as after that the ID will be different
+                                    foreach (var (fromEntityID, _, _) in fromEntityToEntityIDs)
+                                        enginesRoot._cachedIndicesToSwapBeforeSubmissionForFilters.Add(fromEntityID,
+                                            fromComponentsDictionary.GetIndex(fromEntityID));
                                 }
+
+                                //ensure that to dictionary has enough room to store the new entities`
+                                toComponentsDictionary.EnsureCapacity((uint)(toComponentsDictionary.count +
+                                    (uint)fromEntityToEntityIDs.count));
+
+                                //fortunately swap means that entities are added at the end of each destination
+                                //dictionary list, so we can just iterate the list using the indices ranges added in the
+                                //_cachedIndices
+                                enginesRoot._cachedRangeOfSubmittedIndices.Add(((uint, uint))(
+                                    toComponentsDictionary.count,
+                                    toComponentsDictionary.count + fromEntityToEntityIDs.count));
+
+                                fromComponentsDictionary.SwapEntitiesBetweenDictionaries(fromEntityToEntityIDs,
+                                    fromGroup, toGroup, toComponentsDictionary);
+
+                                if (fromDictionaryCountBeforeSubmission != -1) //this if skips the swap if there are no filters linked to the component
+                                    enginesRoot.SwapEntityBetweenPersistentFilters(fromEntityToEntityIDs,
+                                        enginesRoot._cachedIndicesToSwapBeforeSubmissionForFilters,
+                                        toComponentsDictionary, fromGroup, toGroup,
+                                        (uint)fromDictionaryCountBeforeSubmission, listOfFilters);
                             }
-                            catch
+                        }
+                    }
+                }
+
+                using (sampler.Sample("Execute Swap Callbacks"))
+                {
+                    foreach (var entitiesToSwap in swapEntitiesOperations)
+                    {
+                        ExclusiveGroupStruct fromGroup = entitiesToSwap.key;
+
+                        foreach (var groupedEntitiesToSwap in entitiesToSwap.value)
+                        {
+                            var componentType = groupedEntitiesToSwap.key;
+
+                            //get all the engines linked to TValue
+                            if (!enginesRoot._reactiveEnginesSwap.TryGetValue(new RefWrapperType(componentType),
+                                    out var entityComponentsEngines))
+                                continue;
+
+                            foreach (var entitiesInfoToSwap in groupedEntitiesToSwap.value)
                             {
-                                var str = "Crash while executing Entity Operation ".FastConcat(
-                                    entitiesOperations[i].type.ToString());
+                                ExclusiveGroupStruct toGroup = entitiesInfoToSwap.key;
+                                ITypeSafeDictionary toComponentsDictionary = GetTypeSafeDictionary(toGroup,
+                                    enginesRoot.GetDBGroup(toGroup), componentType);
 
-                                Svelto.Console.LogError(str.FastConcat(" ")
-#if DEBUG && !PROFILE_SVELTO
-                                                      .FastConcat(entitiesOperations[i].trace.ToString())
-#endif
-                                );
+                                var infosToProcess = entitiesInfoToSwap.value;
 
-                                throw;
+                                toComponentsDictionary.ExecuteEnginesSwapCallbacks(infosToProcess,
+                                    entityComponentsEngines, fromGroup, toGroup, in sampler);
                             }
+                        }
+                    }
+                }
 
-                            ++numberOfOperations;
+                var rangeEnumerator = enginesRoot._cachedRangeOfSubmittedIndices.GetEnumerator();
+                using (sampler.Sample("Execute Swap Callbacks Fast"))
+                {
+                    foreach (var entitiesToSwap in swapEntitiesOperations)
+                    {
+                        ExclusiveGroupStruct fromGroup = entitiesToSwap.key;
+
+                        foreach (var groupedEntitiesToSwap in entitiesToSwap.value)
+                        {
+                            var componentType = groupedEntitiesToSwap.key;
 
-                            if ((uint) numberOfOperations >= (uint) _maxNumberOfOperationsPerFrame)
+                            foreach (var entitiesInfoToSwap in groupedEntitiesToSwap.value)
                             {
-                                using (sample.Yield())
-                                    yield return true;
+                                rangeEnumerator.MoveNext();
 
-                                numberOfOperations = 0;
+                                //get all the engines linked to TValue
+                                if (!enginesRoot._reactiveEnginesSwapEx.TryGetValue(new RefWrapperType(componentType),
+                                        out var entityComponentsEngines))
+                                    continue;
+
+                                ExclusiveGroupStruct toGroup = entitiesInfoToSwap.key;
+                                ITypeSafeDictionary toComponentsDictionary = GetTypeSafeDictionary(toGroup,
+                                    enginesRoot.GetDBGroup(toGroup), componentType);
+
+                                toComponentsDictionary.ExecuteEnginesSwapCallbacksFast(entityComponentsEngines,
+                                    fromGroup, toGroup, rangeEnumerator.Current, in sampler);
                             }
                         }
                     }
                 }
+            }
+        }
 
-                _groupedEntityToAdd.Swap();
+        void AddEntities(PlatformProfiler sampler)
+        {
+            //current buffer becomes other, and other becomes current
+            _groupedEntityToAdd.Swap();
 
-                if (_groupedEntityToAdd.AnyOtherEntityCreated())
+            //I need to iterate the previous current, which is now other
+            if (_groupedEntityToAdd.AnyPreviousEntityCreated())
+            {
+                _cachedRangeOfSubmittedIndices.FastClear();
+                using (sampler.Sample("Add operations"))
                 {
-                    using (var outerSampler = profiler.Sample("Add operations"))
+                    try
                     {
-                        try
+                        using (sampler.Sample("Add entities to database"))
                         {
-                            using (profiler.Sample("Add entities to database"))
+                            //each group is indexed by entity view type. for each type there is a dictionary indexed
+                            //by entityID
+                            foreach (var groupToSubmit in _groupedEntityToAdd)
                             {
-                                //each group is indexed by entity view type. for each type there is a dictionary indexed by entityID
-                                foreach (var groupToSubmit in _groupedEntityToAdd.other)
+                                var groupID = groupToSubmit.@group;
+                                var groupDB = GetOrAddDBGroup(groupID);
+
+                                //add the entityComponents in the group
+                                foreach (var entityComponentsToSubmit in groupToSubmit.components)
                                 {
-                                    var groupID = groupToSubmit.Key;
-                                    var groupDB = GetOrCreateDBGroup(groupID);
-
-                                    //add the entityComponents in the group
-                                    foreach (var entityComponentsToSubmit in groupToSubmit.Value)
-                                    {
-                                        var type                     = entityComponentsToSubmit.Key;
-                                        var targetTypeSafeDictionary = entityComponentsToSubmit.Value;
-                                        var wrapper                  = new RefWrapperType(type);
-
-                                        var dbDic = GetOrCreateTypeSafeDictionary(
-                                            groupID, groupDB, wrapper, targetTypeSafeDictionary);
-
-                                        //Fill the DB with the entity components generated this frame.
-                                        dbDic.AddEntitiesFromDictionary(targetTypeSafeDictionary, groupID, this);
-                                    }
+                                    var type           = entityComponentsToSubmit.key;
+                                    var fromDictionary = entityComponentsToSubmit.value;
+                                    var wrapper        = new RefWrapperType(type);
+
+                                    var toDictionary =
+                                        GetOrAddTypeSafeDictionary(groupID, groupDB, wrapper, fromDictionary);
+
+                                    //all the new entities are added at the end of each dictionary list, so we can
+                                    //just iterate the list using the indices ranges added in the _cachedIndices
+                                    _cachedRangeOfSubmittedIndices.Add(((uint, uint))(toDictionary.count,
+                                        toDictionary.count + fromDictionary.count));
+                                    //Fill the DB with the entity components generated this frame.
+                                    fromDictionary.AddEntitiesToDictionary(toDictionary, groupID, entityLocator);
                                 }
                             }
+                        }
 
-                            //then submit everything in the engines, so that the DB is up to date with all the entity components
-                            //created by the entity built
-                            using (var sampler = profiler.Sample("Add entities to engines"))
+                        //then submit everything in the engines, so that the DB is up to date with all the entity components
+                        //created by the entity built
+                        var enumerator = _cachedRangeOfSubmittedIndices.GetEnumerator();
+                        using (sampler.Sample("Add entities to engines fast"))
+                        {
+                            foreach (GroupInfo groupToSubmit in _groupedEntityToAdd)
                             {
-                                foreach (var groupToSubmit in _groupedEntityToAdd.other)
+                                var groupID = groupToSubmit.@group;
+                                var groupDB = GetDBGroup(groupID);
+
+                                foreach (var entityComponentsToSubmit in groupToSubmit.components)
                                 {
-                                    var groupID = groupToSubmit.Key;
-                                    var groupDB = GetDBGroup(groupID);
-//entityComponentsToSubmit is the array of components found in the groupID per component type. 
-//if there are N entities to submit, and M components type to add for each entity, this foreach will run NxM times. 
-                                    foreach (var entityComponentsToSubmit in groupToSubmit.Value)
-                                    {
-                                        var realDic = groupDB[new RefWrapperType(entityComponentsToSubmit.Key)];
-
-                                        entityComponentsToSubmit.Value.ExecuteEnginesAddOrSwapCallbacks(
-                                            _reactiveEnginesAddRemove, realDic, null, groupID, in profiler);
-
-                                        numberOfOperations += entityComponentsToSubmit.Value.count;
-
-                                        if (numberOfOperations >= _maxNumberOfOperationsPerFrame)
-                                        {
-                                            using (outerSampler.Yield())
-                                            using (sampler.Yield())
-                                            {
-                                                yield return true;
-                                            }
-
-                                            numberOfOperations = 0;
-                                        }
-                                    }
+                                    var type    = entityComponentsToSubmit.key;
+                                    var wrapper = new RefWrapperType(type);
+
+                                    var toDictionary = GetTypeSafeDictionary(groupID, groupDB, wrapper);
+                                    enumerator.MoveNext();
+                                    toDictionary.ExecuteEnginesAddEntityCallbacksFast(_reactiveEnginesAddEx, groupID,
+                                        enumerator.Current, in sampler);
                                 }
                             }
                         }
-                        finally
+
+                        //then submit everything in the engines, so that the DB is up to date with all the entity components
+                        //created by the entity built
+                        using (sampler.Sample("Add entities to engines"))
                         {
-                            using (profiler.Sample("clear double buffering"))
+                            foreach (GroupInfo groupToSubmit in _groupedEntityToAdd)
                             {
-                                //other can be cleared now, but let's avoid deleting the dictionary every time
-                                _groupedEntityToAdd.ClearOther();
+                                var groupID = groupToSubmit.@group;
+                                var groupDB = GetDBGroup(groupID);
+
+                                //This loop iterates again all the entity components that have been just submitted to call
+                                //the Add Callbacks on them. Note that I am iterating the transient buffer of the just
+                                //added components, but calling the callback on the entities just added in the real buffer
+                                //Note: it's OK to add new entities while this happens because of the double buffer
+                                //design of the transient buffer of added entities.
+                                foreach (var entityComponentsToSubmit in groupToSubmit.components)
+                                {
+                                    var type           = entityComponentsToSubmit.key;
+                                    var fromDictionary = entityComponentsToSubmit.value;
+
+                                    //this contains the total number of components ever submitted in the DB
+                                    ITypeSafeDictionary toDictionary = GetTypeSafeDictionary(groupID, groupDB, type);
+
+                                    fromDictionary.ExecuteEnginesAddCallbacks(_reactiveEnginesAdd, toDictionary,
+                                        groupID, in sampler);
+                                }
                             }
                         }
                     }
+                    finally
+                    {
+                        using (sampler.Sample("clear double buffering"))
+                        {
+                            //other can be cleared now, but let's avoid deleting the dictionary every time
+                            _groupedEntityToAdd.ClearLastAddOperations();
+                        }
+                    }
                 }
-
-                yield return false;
             }
         }
-        
+
         bool HasMadeNewStructuralChangesInThisIteration()
         {
-            return _groupedEntityToAdd.AnyEntityCreated() || _entitiesOperations.count > 0;
+            return _groupedEntityToAdd.AnyEntityCreated() || _entitiesOperations.AnyOperationQueued();
         }
 
-        readonly DoubleBufferedEntitiesToAdd                    _groupedEntityToAdd;
-        readonly FasterDictionary<ulong, EntitySubmitOperation> _entitiesOperations;
-        readonly FasterList<EntitySubmitOperation>              _transientEntitiesOperations;
-        uint                                                    _maxNumberOfOperationsPerFrame;
+        void RemoveEntitiesFromGroup(ExclusiveGroupStruct groupID, in PlatformProfiler profiler)
+        {
+            _entityLocator.RemoveAllGroupReferenceLocators(groupID);
+
+            if (_groupEntityComponentsDB.TryGetValue(groupID, out var dictionariesOfEntities))
+            {
+                foreach (var dictionaryOfEntities in dictionariesOfEntities)
+                {
+                    //RemoveEX happens inside
+                    dictionaryOfEntities.value.ExecuteEnginesRemoveCallbacks_Group(_reactiveEnginesRemove, 
+                        _reactiveEnginesRemoveEx, groupID, profiler);
+                }
+
+                foreach (var dictionaryOfEntities in dictionariesOfEntities)
+                {
+                    dictionaryOfEntities.value.Clear();
+
+                    _groupsPerEntity[dictionaryOfEntities.key][groupID].Clear();
+                }
+            }
+        }
+
+        void SwapEntitiesBetweenGroups(ExclusiveGroupStruct fromGroupId, ExclusiveGroupStruct toGroupId,
+            PlatformProfiler platformProfiler)
+        {
+            FasterDictionary<RefWrapperType, ITypeSafeDictionary> fromGroup = GetDBGroup(fromGroupId);
+            FasterDictionary<RefWrapperType, ITypeSafeDictionary> toGroup   = GetOrAddDBGroup(toGroupId);
+
+            _entityLocator.UpdateAllGroupReferenceLocators(fromGroupId, toGroupId);
+
+            //remove entities from dictionaries
+            foreach (var dictionaryOfEntities in fromGroup)
+            {
+                RefWrapperType refWrapperType = dictionaryOfEntities.key;
+
+                ITypeSafeDictionary fromDictionary = dictionaryOfEntities.value;
+                ITypeSafeDictionary toDictionary =
+                    GetOrAddTypeSafeDictionary(toGroupId, toGroup, refWrapperType, fromDictionary);
+
+                fromDictionary.AddEntitiesToDictionary(toDictionary, toGroupId, this.entityLocator);
+            }
+
+            //Call all the callbacks
+            foreach (var dictionaryOfEntities in fromGroup)
+            {
+                RefWrapperType refWrapperType = dictionaryOfEntities.key;
+
+                ITypeSafeDictionary fromDictionary = dictionaryOfEntities.value;
+                ITypeSafeDictionary toDictionary   = GetTypeSafeDictionary(toGroupId, toGroup, refWrapperType);
+
+                //SwapEX happens inside
+                fromDictionary.ExecuteEnginesSwapCallbacks_Group(_reactiveEnginesSwap, 
+                    _reactiveEnginesSwapEx, toDictionary, fromGroupId, toGroupId, platformProfiler);
+            }
+
+            //remove entities from dictionaries
+            foreach (var dictionaryOfEntities in fromGroup)
+            {
+                dictionaryOfEntities.value.Clear();
+
+                _groupsPerEntity[dictionaryOfEntities.key][fromGroupId].Clear();
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        ITypeSafeDictionary GetOrAddTypeSafeDictionary(ExclusiveGroupStruct groupId,
+            FasterDictionary<RefWrapperType, ITypeSafeDictionary> groupPerComponentType, RefWrapperType type,
+            ITypeSafeDictionary fromDictionary)
+        {
+            //be sure that the TypeSafeDictionary for the entity Type exists
+            if (groupPerComponentType.TryGetValue(type, out ITypeSafeDictionary toEntitiesDictionary) == false)
+            {
+                toEntitiesDictionary = fromDictionary.Create();
+                groupPerComponentType.Add(type, toEntitiesDictionary);
+            }
+
+            {
+                //update GroupsPerEntity
+                if (_groupsPerEntity.TryGetValue(type, out var groupedGroup) == false)
+                    groupedGroup = _groupsPerEntity[type] =
+                        new FasterDictionary<ExclusiveGroupStruct, ITypeSafeDictionary>();
+
+                groupedGroup[groupId] = toEntitiesDictionary;
+            }
+
+            return toEntitiesDictionary;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        static ITypeSafeDictionary GetTypeSafeDictionary(ExclusiveGroupStruct groupID,
+            FasterDictionary<RefWrapperType, ITypeSafeDictionary> @group, RefWrapperType refWrapper)
+        {
+            if (@group.TryGetValue(refWrapper, out ITypeSafeDictionary fromTypeSafeDictionary) == false)
+            {
+                throw new ECSException("no group found: ".FastConcat(groupID.ToName()));
+            }
+
+            return fromTypeSafeDictionary;
+        }
+
+        readonly DoubleBufferedEntitiesToAdd  _groupedEntityToAdd;
+        readonly EntitiesOperations           _entitiesOperations;
+        readonly FasterList<(uint, uint)>     _cachedRangeOfSubmittedIndices;
+        readonly FasterDictionary<uint, uint> _cachedIndicesToSwapBeforeSubmissionForFilters;
+
+        static readonly
+            Action<FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType,
+                    FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>>>, FasterList<(EGID, EGID)>
+               ,
+                EnginesRoot> _swapEntities;
+
+        static readonly Action<
+            FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, FasterList<(uint, string)>>>,
+            FasterList<EGID>, EnginesRoot> _removeEntities;
+
+        static readonly Action<ExclusiveGroupStruct, EnginesRoot>                       _removeGroup;
+        static readonly Action<ExclusiveGroupStruct, ExclusiveGroupStruct, EnginesRoot> _swapGroup;
     }
 }

Core/EntitiesDB.FindGroups.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Runtime.CompilerServices;
 using System.Threading;
 using Svelto.DataStructures;
 using Svelto.ECS.Internal;
@@ -115,9 +114,9 @@ namespace Svelto.ECS
 
             foreach (var value in localArray[startIndex])
             {
-                if (value.Key.IsEnabled())
+                if (value.key.IsEnabled())
                 {
-                    localGroups.Add(value.Key, value.Key);
+                    localGroups.Add(value.key, value.key);
                 }
             }
 
@@ -171,9 +170,9 @@ namespace Svelto.ECS
 
             foreach (var value in localArray[startIndex])
             {
-                if (value.Key.IsEnabled())
+                if (value.key.IsEnabled())
                 {
-                    localGroups.Add(value.Key, value.Key);
+                    localGroups.Add(value.key, value.key);
                 }
             }
 

Core/EntitiesDB.cs

@@ -6,14 +6,13 @@ using System;
 using System.Runtime.CompilerServices;
 using Svelto.Common;
 using Svelto.DataStructures;
-using Svelto.DataStructures.Native;
 using Svelto.ECS.Internal;
 
 namespace Svelto.ECS
 {
     public partial class EntitiesDB
     {
-        internal EntitiesDB(EnginesRoot enginesRoot, EnginesRoot.LocatorMap entityReferencesMap)
+        internal EntitiesDB(EnginesRoot enginesRoot, EnginesRoot.EntityReferenceMap entityReferencesMap)
         {
             _enginesRoot   = enginesRoot;
             _entityReferencesMap = entityReferencesMap;
@@ -25,15 +24,17 @@ namespace Svelto.ECS
         {
             uint       count = 0;
             IBuffer<T> buffer;
+            EntityIDs  ids = default;
             if (SafeQueryEntityDictionary<T>(out var typeSafeDictionary, entitiesInGroupPerType) == false)
                 buffer = RetrieveEmptyEntityComponentArray<T>();
             else
             {
-                var safeDictionary = (typeSafeDictionary as ITypeSafeDictionary<T>);
+                ITypeSafeDictionary<T> safeDictionary = (typeSafeDictionary as ITypeSafeDictionary<T>);
                 buffer = safeDictionary.GetValues(out count);
+                ids    = safeDictionary.entityIDs;
             }
 
-            return new EntityCollection<T>(buffer, count);
+            return new EntityCollection<T>(buffer, count, ids);
         }
 
         /// <summary>
@@ -257,7 +258,7 @@ namespace Svelto.ECS
         public bool IsDisposing => _enginesRoot._isDisposing;
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal bool SafeQueryEntityDictionary<T>
+        bool SafeQueryEntityDictionary<T>
         (out ITypeSafeDictionary typeSafeDictionary
        , FasterDictionary<RefWrapperType, ITypeSafeDictionary> entitiesInGroupPerType) where T : IEntityComponent
         {
@@ -353,6 +354,6 @@ namespace Svelto.ECS
         FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, ITypeSafeDictionary>> groupsPerEntity =>
             _enginesRoot._groupsPerEntity;
 
-        EnginesRoot.LocatorMap _entityReferencesMap;
+        EnginesRoot.EntityReferenceMap _entityReferencesMap;
     }
 }

Core/EntitiesOperations.cs

@@ -0,0 +1,191 @@
+using System;
+using Svelto.DataStructures;
+
+namespace Svelto.ECS
+{
+    class EntitiesOperations
+    {
+        public EntitiesOperations()
+        {
+            _thisSubmissionInfo.Init();
+            _lastSubmittedInfo.Init();
+        }
+
+        public void AddSwapOperation(EGID fromID, EGID toID, IComponentBuilder[] componentBuilders, string caller)
+        {
+            _thisSubmissionInfo._entitiesSwapped.Add((fromID, toID));
+
+            //todo: limit the number of dictionaries that can be cached 
+            //recycle or create dictionaries of components per group
+            var swappedComponentsPerType = _thisSubmissionInfo._currentSwapEntitiesOperations.RecycleOrAdd(
+                fromID.groupID,
+                () => new FasterDictionary<RefWrapperType,
+                    FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>>(),
+                (ref FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>> recycled) =>
+                    recycled.FastClear());
+
+            foreach (IComponentBuilder operation in componentBuilders)
+            {
+                swappedComponentsPerType
+                    //recycle or create dictionaries per component type
+                   .RecycleOrAdd(new RefWrapperType(operation.GetEntityComponentType()),
+                        () => new FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>(),
+                        (ref FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>> target) =>
+                            target.FastClear())
+                    //recycle or create list of entities to swap
+                   .RecycleOrAdd(toID.groupID, () => new FasterList<(uint, uint, string)>(),
+                        (ref FasterList<(uint, uint, string)> target) => target.FastClear())
+                    //add entity to swap
+                   .Add((fromID.entityID, toID.entityID, caller));
+            }
+        }
+
+        public void AddRemoveOperation(EGID entityEgid, IComponentBuilder[] componentBuilders, string caller)
+        {
+            _thisSubmissionInfo._entitiesRemoved.Add(entityEgid);
+            //todo: limit the number of dictionaries that can be cached 
+            //recycle or create dictionaries of components per group
+            var removedComponentsPerType = _thisSubmissionInfo._currentRemoveEntitiesOperations.RecycleOrAdd(
+                entityEgid.groupID, () => new FasterDictionary<RefWrapperType, FasterList<(uint, string)>>(),
+                (ref FasterDictionary<RefWrapperType, FasterList<(uint, string)>> recycled) => recycled.FastClear());
+
+            foreach (IComponentBuilder operation in componentBuilders)
+            {
+                removedComponentsPerType
+                    //recycle or create dictionaries per component type
+                   .RecycleOrAdd(new RefWrapperType(operation.GetEntityComponentType()),
+                        () => new FasterList<(uint, string)>(),
+                        (ref FasterList<(uint, string)> target) => target.FastClear())
+                    //add entity to swap
+                   .Add((entityEgid.entityID, caller));
+            }
+        }
+
+        public void AddRemoveGroupOperation(ExclusiveBuildGroup groupID, string caller)
+        {
+            _thisSubmissionInfo._groupsToRemove.Add((groupID, caller));
+        }
+
+        public void AddSwapGroupOperation(ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID, string caller)
+        {
+            _thisSubmissionInfo._groupsToSwap.Add((fromGroupID, toGroupID, caller));
+        }
+
+        public void ExecuteRemoveAndSwappingOperations(
+            Action<FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType,
+                    FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>>>, FasterList<(EGID, EGID)>
+               ,
+                EnginesRoot> swapEntities,
+            Action<FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType, FasterList<(uint, string)>>>,
+                FasterList<EGID>, EnginesRoot> removeEntities, Action<ExclusiveGroupStruct, EnginesRoot> removeGroup,
+            Action<ExclusiveGroupStruct, ExclusiveGroupStruct, EnginesRoot> swapGroup, EnginesRoot enginesRoot)
+        {
+            (_thisSubmissionInfo, _lastSubmittedInfo) = (_lastSubmittedInfo, _thisSubmissionInfo);
+
+            ///todo: we found a case where entities with reference to other entities were removed
+            /// in the same frame where the referenced entities are remove too.
+            /// the callback of the referencing entities were assuming that the reference at that point
+            /// would be invalid. However since the callbacks were called before the groups are removed
+            /// the reference were still valid, which was not expected.
+            /// If the referenced entities were removed one by one instead that with the group, by chance
+            /// it instead worked because the entities were removed before the callbacks were called.
+            /// this is why RemoveGroup is happeing before RemoveEntities, however the real fix
+            /// should be to update all the references before removing the entities from the dictionaries
+            /// and call the callbacks
+            foreach (var (group, caller) in _lastSubmittedInfo._groupsToRemove)
+            {
+                try
+                {
+                    removeGroup(group, enginesRoot);
+                }
+                catch
+                {
+                    var str = "Crash while removing a whole group on ".FastConcat(group.ToString())
+                       .FastConcat(" from : ", caller);
+
+                    Console.LogError(str);
+
+                    throw;
+                }
+            }
+
+            foreach (var (fromGroup, toGroup, caller) in _lastSubmittedInfo._groupsToSwap)
+            {
+                try
+                {
+                    swapGroup(fromGroup, toGroup, enginesRoot);
+                }
+                catch
+                {
+                    var str = "Crash while swapping a whole group on "
+                       .FastConcat(fromGroup.ToString(), " ", toGroup.ToString()).FastConcat(" from : ", caller);
+
+                    Console.LogError(str);
+
+                    throw;
+                }
+            }
+
+            if (_lastSubmittedInfo._entitiesSwapped.count > 0)
+                swapEntities(_lastSubmittedInfo._currentSwapEntitiesOperations, _lastSubmittedInfo._entitiesSwapped,
+                    enginesRoot);
+
+            if (_lastSubmittedInfo._entitiesRemoved.count > 0)
+                removeEntities(_lastSubmittedInfo._currentRemoveEntitiesOperations, _lastSubmittedInfo._entitiesRemoved,
+                    enginesRoot);
+
+            _lastSubmittedInfo.Clear();
+        }
+
+        public bool AnyOperationQueued() => _thisSubmissionInfo.AnyOperationQueued();
+
+        struct Info
+        {
+            //from group                          //actual component type      
+            internal FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType,
+                    // to group ID        //entityIDs , debugInfo
+                    FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>>>
+                _currentSwapEntitiesOperations;
+
+            internal FasterDictionary<ExclusiveGroupStruct,
+                FasterDictionary<RefWrapperType, FasterList<(uint, string)>>> _currentRemoveEntitiesOperations;
+
+            internal FasterList<(EGID, EGID)>                                       _entitiesSwapped;
+            internal FasterList<EGID>                                               _entitiesRemoved;
+            public   FasterList<(ExclusiveBuildGroup, ExclusiveBuildGroup, string)> _groupsToSwap;
+            public   FasterList<(ExclusiveBuildGroup, string)>                      _groupsToRemove;
+
+            internal bool AnyOperationQueued() =>
+                _entitiesSwapped.count > 0 || _entitiesRemoved.count > 0 || _groupsToSwap.count > 0 ||
+                _groupsToRemove.count > 0;
+
+            internal void Clear()
+            {
+                _currentSwapEntitiesOperations.FastClear();
+                _currentRemoveEntitiesOperations.FastClear();
+                _entitiesSwapped.FastClear();
+                _entitiesRemoved.FastClear();
+                _groupsToRemove.FastClear();
+                _groupsToSwap.FastClear();
+            }
+
+            internal void Init()
+            {
+                _entitiesSwapped = new FasterList<(EGID, EGID)>();
+                _entitiesRemoved = new FasterList<EGID>();
+                _groupsToRemove  = new FasterList<(ExclusiveBuildGroup, string)>();
+                _groupsToSwap    = new FasterList<(ExclusiveBuildGroup, ExclusiveBuildGroup, string)>();
+
+                _currentSwapEntitiesOperations =
+                    new FasterDictionary<ExclusiveGroupStruct, FasterDictionary<RefWrapperType,
+                        FasterDictionary<ExclusiveGroupStruct, FasterList<(uint, uint, string)>>>>();
+                _currentRemoveEntitiesOperations =
+                    new FasterDictionary<ExclusiveGroupStruct,
+                        FasterDictionary<RefWrapperType, FasterList<(uint, string)>>>();
+            }
+        }
+
+        Info _thisSubmissionInfo;
+        Info _lastSubmittedInfo;
+    }
+}

Components/LinkedEntityComponent.cs.meta → Core/EntitiesOperations.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: b2536af786e0381aa68a66f6569358bf
+guid: e06e09d6532f3bf38d9940602638b084
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Core/EntityCollection.cs

@@ -1,6 +1,5 @@
 using System.Runtime.CompilerServices;
 using Svelto.DataStructures;
-using Svelto.DataStructures.Native;
 using Svelto.ECS.Internal;
 
 namespace Svelto.ECS
@@ -9,134 +8,139 @@ namespace Svelto.ECS
     {
         static readonly bool IsUnmanaged = TypeSafeDictionary<T>.isUnmanaged;
 
+        public EntityCollection(IBuffer<T> buffer, uint count, EntityIDs entityIDs) : this()
+        {
+            DBC.ECS.Check.Require(count == 0 || buffer.isValid, "Buffer is found in impossible state");
+            if (IsUnmanaged)
+            {
+                _nativedBuffer  = (NB<T>)buffer;
+                _nativedIndices = entityIDs.nativeIDs;
+            }
+            else
+            {
+                _managedBuffer  = (MB<T>)buffer;
+                _managedIndices = entityIDs.managedIDs;
+            }
+
+            this.count          = count;
+        }
+        
         public EntityCollection(IBuffer<T> buffer, uint count) : this()
         {
             DBC.ECS.Check.Require(count == 0 || buffer.isValid, "Buffer is found in impossible state");
             if (IsUnmanaged)
-                _nativedBuffer = (NB<T>) buffer;
+                _nativedBuffer = (NB<T>)buffer;
             else
-                _managedBuffer = (MB<T>) buffer;
+                _managedBuffer = (MB<T>)buffer;
 
-            _count = count;
+            this.count = count;
         }
 
-        public uint count => _count;
+        public uint count { get; }
 
         internal readonly MB<T> _managedBuffer;
         internal readonly NB<T> _nativedBuffer;
-
-        readonly uint _count;
+        
+        internal readonly NativeEntityIDs  _nativedIndices;
+        internal readonly ManagedEntityIDs _managedIndices;
     }
 
-    public readonly ref struct EntityCollection<T1, T2>
-        where T1 : struct, IEntityComponent where T2 : struct, IEntityComponent
+    public readonly ref struct EntityCollection<T1, T2> where T1 : struct, IEntityComponent
+        where T2 : struct, IEntityComponent
     {
         internal EntityCollection(in EntityCollection<T1> array1, in EntityCollection<T2> array2)
         {
-            _array1 = array1;
-            _array2 = array2;
+            buffer1 = array1;
+            buffer2 = array2;
         }
 
-        public uint count => _array1.count;
+        public int count => (int)buffer1.count;
 
         internal EntityCollection<T2> buffer2
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array2;
+            get;
         }
 
         internal EntityCollection<T1> buffer1
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array1;
+            get;
         }
-
-        readonly EntityCollection<T1> _array1;
-        readonly EntityCollection<T2> _array2;
     }
 
     public readonly ref struct EntityCollection<T1, T2, T3> where T3 : struct, IEntityComponent
-                                                            where T2 : struct, IEntityComponent
-                                                            where T1 : struct, IEntityComponent
+        where T2 : struct, IEntityComponent
+        where T1 : struct, IEntityComponent
     {
-        internal EntityCollection
-            (in EntityCollection<T1> array1, in EntityCollection<T2> array2, in EntityCollection<T3> array3)
+        internal EntityCollection(in EntityCollection<T1> array1, in EntityCollection<T2> array2,
+            in EntityCollection<T3> array3)
         {
-            _array1 = array1;
-            _array2 = array2;
-            _array3 = array3;
+            buffer1 = array1;
+            buffer2 = array2;
+            buffer3 = array3;
         }
 
         internal EntityCollection<T1> buffer1
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array1;
+            get;
         }
 
         internal EntityCollection<T2> buffer2
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array2;
+            get;
         }
 
         internal EntityCollection<T3> buffer3
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array3;
+            get;
         }
 
-        internal uint count => buffer1.count;
-
-        readonly EntityCollection<T1> _array1;
-        readonly EntityCollection<T2> _array2;
-        readonly EntityCollection<T3> _array3;
+        internal int count => (int)buffer1.count;
     }
 
     public readonly ref struct EntityCollection<T1, T2, T3, T4> where T1 : struct, IEntityComponent
-                                                                where T2 : struct, IEntityComponent
-                                                                where T3 : struct, IEntityComponent
-                                                                where T4 : struct, IEntityComponent
+        where T2 : struct, IEntityComponent
+        where T3 : struct, IEntityComponent
+        where T4 : struct, IEntityComponent
     {
-        internal EntityCollection
-        (in EntityCollection<T1> array1, in EntityCollection<T2> array2, in EntityCollection<T3> array3
-       , in EntityCollection<T4> array4)
+        internal EntityCollection(in EntityCollection<T1> array1, in EntityCollection<T2> array2,
+            in EntityCollection<T3> array3, in EntityCollection<T4> array4)
         {
-            _array1 = array1;
-            _array2 = array2;
-            _array3 = array3;
-            _array4 = array4;
+            buffer1 = array1;
+            buffer2 = array2;
+            buffer3 = array3;
+            buffer4 = array4;
         }
 
         internal EntityCollection<T1> buffer1
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array1;
+            get;
         }
 
         internal EntityCollection<T2> buffer2
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array2;
+            get;
         }
 
         internal EntityCollection<T3> buffer3
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array3;
+            get;
         }
 
         internal EntityCollection<T4> buffer4
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get => _array4;
+            get;
         }
 
-        internal uint count => _array1.count;
-
-        readonly EntityCollection<T1> _array1;
-        readonly EntityCollection<T2> _array2;
-        readonly EntityCollection<T3> _array3;
-        readonly EntityCollection<T4> _array4;
+        internal int count => (int)buffer1.count;
     }
 
     public readonly struct BT<BufferT1, BufferT2, BufferT3, BufferT4>
@@ -147,13 +151,20 @@ namespace Svelto.ECS
         public readonly BufferT4 buffer4;
         public readonly int      count;
 
-        public BT(BufferT1 bufferT1, BufferT2 bufferT2, BufferT3 bufferT3, BufferT4 bufferT4, uint count) : this()
+        BT(in (BufferT1 bufferT1, BufferT2 bufferT2, BufferT3 bufferT3, BufferT4 bufferT4, int count) buffer) :
+            this()
+        {
+            buffer1 = buffer.bufferT1;
+            buffer2 = buffer.bufferT2;
+            buffer3 = buffer.bufferT3;
+            buffer4 = buffer.bufferT4;
+            count   = buffer.count;
+        }
+
+        public static implicit operator BT<BufferT1, BufferT2, BufferT3, BufferT4>(
+            in (BufferT1 bufferT1, BufferT2 bufferT2, BufferT3 bufferT3, BufferT4 bufferT4, int count) buffer)
         {
-            this.buffer1 = bufferT1;
-            this.buffer2 = bufferT2;
-            this.buffer3 = bufferT3;
-            this.buffer4 = bufferT4;
-            this.count   = (int) count;
+            return new BT<BufferT1, BufferT2, BufferT3, BufferT4>(buffer);
         }
     }
 
@@ -164,20 +175,18 @@ namespace Svelto.ECS
         public readonly BufferT3 buffer3;
         public readonly int      count;
 
-        public BT(BufferT1 bufferT1, BufferT2 bufferT2, BufferT3 bufferT3, uint count) : this()
+        BT(in (BufferT1 bufferT1, BufferT2 bufferT2, BufferT3 bufferT3, int count) buffer) : this()
         {
-            this.buffer1 = bufferT1;
-            this.buffer2 = bufferT2;
-            this.buffer3 = bufferT3;
-            this.count   = (int) count;
+            buffer1 = buffer.bufferT1;
+            buffer2 = buffer.bufferT2;
+            buffer3 = buffer.bufferT3;
+            count   = buffer.count;
         }
 
-        public void Deconstruct(out BufferT1 bufferT1, out BufferT2 bufferT2, out BufferT3 bufferT3, out int count)
+        public static implicit operator BT<BufferT1, BufferT2, BufferT3>(
+            in (BufferT1 bufferT1, BufferT2 bufferT2, BufferT3 bufferT3, int count) buffer)
         {
-            bufferT1 = buffer1;
-            bufferT2 = buffer2;
-            bufferT3 = buffer3;
-            count    = this.count;
+            return new BT<BufferT1, BufferT2, BufferT3>(buffer);
         }
     }
 
@@ -186,16 +195,15 @@ namespace Svelto.ECS
         public readonly BufferT1 buffer;
         public readonly int      count;
 
-        public BT(BufferT1 bufferT1, uint count) : this()
+        BT(in (BufferT1 bufferT1, int count) buffer) : this()
         {
-            this.buffer = bufferT1;
-            this.count  = (int) count;
+            this.buffer = buffer.bufferT1;
+            count  = buffer.count;
         }
 
-        public void Deconstruct(out BufferT1 bufferT1, out int count)
+        public static implicit operator BT<BufferT1>(in (BufferT1 bufferT1, int count) buffer)
         {
-            bufferT1 = buffer;
-            count    = this.count;
+            return new BT<BufferT1>(buffer);
         }
 
         public static implicit operator BufferT1(BT<BufferT1> t) => t.buffer;
@@ -207,18 +215,17 @@ namespace Svelto.ECS
         public readonly BufferT2 buffer2;
         public readonly int      count;
 
-        public BT(BufferT1 bufferT1, BufferT2 bufferT2, uint count) : this()
+        BT(in (BufferT1 bufferT1, BufferT2 bufferT2, int count) buffer) : this()
         {
-            this.buffer1 = bufferT1;
-            this.buffer2 = bufferT2;
-            this.count   = (int) count;
+            buffer1 = buffer.bufferT1;
+            buffer2 = buffer.bufferT2;
+            count   = buffer.count;
         }
 
-        public void Deconstruct(out BufferT1 bufferT1, out BufferT2 bufferT2, out int count)
+        public static implicit operator BT<BufferT1, BufferT2>(
+            in (BufferT1 bufferT1, BufferT2 bufferT2, int count) buffer)
         {
-            bufferT1 = buffer1;
-            bufferT2 = buffer2;
-            count    = this.count;
+            return new BT<BufferT1, BufferT2>(buffer);
         }
     }
 }

Core/EntityDescriptor/DynamicEntityDescriptor.cs

@@ -129,7 +129,7 @@ namespace Svelto.ECS
 
                     uint index = 0;
                     foreach (var couple in xtraComponents)
-                        newComponents[index++] = couple.Key.value;
+                        newComponents[index++] = couple.key.type;
                 }
 
                 IComponentBuilder[] componentBuilders =

Core/EntityDescriptor/EntityDescriptorExtension.cs

@@ -4,8 +4,8 @@ namespace Svelto.ECS
     {
         public static bool IsUnmanaged(this IEntityDescriptor descriptor)
         {
-            foreach (var component in descriptor.componentsToBuild)
-                if (component.isUnmanaged == false)
+            foreach (IComponentBuilder component in descriptor.componentsToBuild)
+                if (component.GetEntityComponentType() != typeof(EntityInfoComponent) && component.isUnmanaged == false)
                     return false;
             
             return true;

Core/EntityDescriptor/ExtendibleEntityDescriptor.cs

@@ -1,6 +1,3 @@
-using System;
-using Svelto.ECS.Serialization;
-
 namespace Svelto.ECS
 {
     /// <summary>
@@ -25,9 +22,11 @@ namespace Svelto.ECS
     {
         static ExtendibleEntityDescriptor()
         {
-            if (typeof(ISerializableEntityDescriptor).IsAssignableFrom(typeof(TType)))
-                throw new Exception(
-                    $"SerializableEntityDescriptors cannot be used as base entity descriptor: {typeof(TType)}");
+            //I am removing this check because in reality there is not a strong reason to forbid it and
+            //furthermore it's already possible to extend a SerializableEntityDescriptor through DynamicEntityDescriptor
+            // if (typeof(ISerializableEntityDescriptor).IsAssignableFrom(typeof(TType)))
+            //     throw new Exception(
+            //         $"SerializableEntityDescriptors cannot be used as base entity descriptor: {typeof(TType)}");
         }
 
         protected ExtendibleEntityDescriptor(IComponentBuilder[] extraEntities)
@@ -54,7 +53,6 @@ namespace Svelto.ECS
             return this;
         }
 
-
         protected void Add<T>() where T : struct, IEntityComponent
         {
             _dynamicDescriptor.Add<T>();

Core/EntityDescriptor/GenericEntityDescriptor.cs

@@ -2,7 +2,7 @@
 {
     public abstract class GenericEntityDescriptor<T> : IEntityDescriptor where T : struct,  IEntityComponent
     {
-        internal static readonly IComponentBuilder[] _componentBuilders;
+        static readonly IComponentBuilder[] _componentBuilders;
         static GenericEntityDescriptor() { _componentBuilders = new IComponentBuilder[] {new ComponentBuilder<T>()}; }
 
         public IComponentBuilder[] componentsToBuild => _componentBuilders;
@@ -11,7 +11,7 @@
     public abstract class GenericEntityDescriptor<T, U> : IEntityDescriptor
         where T : struct,  IEntityComponent where U : struct,  IEntityComponent
     {
-        internal static readonly IComponentBuilder[] _componentBuilders;
+        static readonly IComponentBuilder[] _componentBuilders;
 
         static GenericEntityDescriptor()
         {
@@ -24,7 +24,7 @@
     public abstract class GenericEntityDescriptor<T, U, V> : IEntityDescriptor
         where T : struct,  IEntityComponent where U : struct,  IEntityComponent where V : struct,  IEntityComponent
     {
-        internal static readonly IComponentBuilder[] _componentBuilders;
+        static readonly IComponentBuilder[] _componentBuilders;
 
         static GenericEntityDescriptor()
         {

Core/EntityFactory.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using Svelto.DataStructures;
 
@@ -10,11 +9,15 @@ namespace Svelto.ECS.Internal
         (EGID egid, EnginesRoot.DoubleBufferedEntitiesToAdd groupEntitiesToAdd, IComponentBuilder[] componentsToBuild
        , IEnumerable<object> implementors
 #if DEBUG && !PROFILE_SVELTO
-       , Type descriptorType
+       , System.Type descriptorType
 #endif
         )
         {
-            var group = FetchEntityGroup(egid.groupID, groupEntitiesToAdd);
+            var group = groupEntitiesToAdd.currentComponentsToAddPerGroup.GetOrAdd(
+                egid.groupID, () => new FasterDictionary<RefWrapperType, ITypeSafeDictionary>());
+
+            //track the number of entities created so far in the group.
+            groupEntitiesToAdd.IncrementEntityCount(egid.groupID);
 
             BuildEntitiesAndAddToGroup(egid, group, componentsToBuild, implementors
 #if DEBUG && !PROFILE_SVELTO
@@ -25,23 +28,11 @@ namespace Svelto.ECS.Internal
             return group;
         }
 
-        static FasterDictionary<RefWrapperType, ITypeSafeDictionary> FetchEntityGroup
-            (ExclusiveGroupStruct groupID, EnginesRoot.DoubleBufferedEntitiesToAdd groupEntityComponentsByType)
-        {
-            var group = groupEntityComponentsByType.current.GetOrCreate(
-                groupID, () => new FasterDictionary<RefWrapperType, ITypeSafeDictionary>());
-
-            //track the number of entities created so far in the group.
-            groupEntityComponentsByType.IncrementEntityCount(groupID);
-
-            return group;
-        }
-
         static void BuildEntitiesAndAddToGroup
         (EGID entityID, FasterDictionary<RefWrapperType, ITypeSafeDictionary> @group
        , IComponentBuilder[] componentBuilders, IEnumerable<object> implementors
 #if DEBUG && !PROFILE_SVELTO
-       , Type descriptorType
+       , System.Type descriptorType
 #endif
         )
         {
@@ -51,7 +42,7 @@ namespace Svelto.ECS.Internal
             var numberOfComponents = componentBuilders.Length;
 
 #if DEBUG && !PROFILE_SVELTO
-            HashSet<Type> types = new HashSet<Type>();
+            var types = new HashSet<System.Type>();
 
             for (var index = 0; index < numberOfComponents; ++index)
             {
@@ -78,7 +69,7 @@ namespace Svelto.ECS.Internal
        , IComponentBuilder componentBuilder, IEnumerable<object> implementors)
         {
             var entityComponentType = componentBuilder.GetEntityComponentType();
-            var safeDictionary = group.GetOrCreate(new RefWrapperType(entityComponentType)
+            ITypeSafeDictionary safeDictionary = group.GetOrAdd(new RefWrapperType(entityComponentType)
                                                  , (ref IComponentBuilder cb) => cb.CreateDictionary(1)
                                                  , ref componentBuilder);
 

Core/EntityInitializer.cs

@@ -1,58 +1,58 @@
 using Svelto.DataStructures;
 using Svelto.ECS.Internal;
-using Svelto.ECS.Reference;
 
 namespace Svelto.ECS
 {
     public readonly ref struct EntityInitializer
     {
-        public EntityInitializer
-            (EGID id, FasterDictionary<RefWrapperType, ITypeSafeDictionary> group, in EntityReference reference)
+        public EntityInitializer(EGID id, FasterDictionary<RefWrapperType, ITypeSafeDictionary> group,
+            in EntityReference reference)
         {
             _group         = group;
             _ID            = id;
             this.reference = reference;
         }
 
-        public   EGID            EGID => _ID;
+        public          EGID            EGID => _ID;
         public readonly EntityReference reference;
 
         public void Init<T>(T initializer) where T : struct, IEntityComponent
         {
-            if (_group.TryGetValue(new RefWrapperType(ComponentBuilder<T>.ENTITY_COMPONENT_TYPE)
-                                 , out var typeSafeDictionary) == false)
+            if (_group.TryGetValue(new RefWrapperType(ComponentBuilder<T>.ENTITY_COMPONENT_TYPE),
+                    out var typeSafeDictionary) == false)
                 return;
 
-            var dictionary = (ITypeSafeDictionary<T>) typeSafeDictionary;
-
+            var dictionary = (ITypeSafeDictionary<T>)typeSafeDictionary;
+#if SLOW_SVELTO_SUBMISSION
             if (ComponentBuilder<T>.HAS_EGID)
                 SetEGIDWithoutBoxing<T>.SetIDWithoutBoxing(ref initializer, _ID);
+#endif
 
             if (dictionary.TryFindIndex(_ID.entityID, out var findElementIndex))
                 dictionary.GetDirectValueByRef(findElementIndex) = initializer;
         }
 
-        public ref T GetOrCreate<T>() where T : struct, IEntityComponent
+        public ref T GetOrAdd<T>() where T : struct, IEntityComponent
         {
-            ref var entityDictionary = ref _group.GetOrCreate(
+            ref var entityDictionary = ref _group.GetOrAdd(
                 new RefWrapperType(ComponentBuilder<T>.ENTITY_COMPONENT_TYPE), TypeSafeDictionaryFactory<T>.Create);
-            var dictionary = (ITypeSafeDictionary<T>) entityDictionary;
+            var dictionary = (ITypeSafeDictionary<T>)entityDictionary;
 
-            return ref dictionary.GetOrCreate(_ID.entityID);
+            return ref dictionary.GetOrAdd(_ID.entityID);
         }
 
         public ref T Get<T>() where T : struct, IEntityComponent
         {
             return ref (_group[new RefWrapperType(ComponentBuilder<T>.ENTITY_COMPONENT_TYPE)] as ITypeSafeDictionary<T>)
-                [_ID.entityID];
+               .GetValueByRef(_ID.entityID);
         }
 
         public bool Has<T>() where T : struct, IEntityComponent
         {
-            if (_group.TryGetValue(new RefWrapperType(ComponentBuilder<T>.ENTITY_COMPONENT_TYPE)
-                                 , out var typeSafeDictionary))
+            if (_group.TryGetValue(new RefWrapperType(ComponentBuilder<T>.ENTITY_COMPONENT_TYPE),
+                    out var typeSafeDictionary))
             {
-                var dictionary = (ITypeSafeDictionary<T>) typeSafeDictionary;
+                var dictionary = (ITypeSafeDictionary<T>)typeSafeDictionary;
 
                 if (dictionary.ContainsKey(_ID.entityID))
                     return true;

Core/EntityReference/EnginesRoot.LocatorMap.cs

@@ -1,6 +1,5 @@
 using System.Runtime.CompilerServices;
 using Svelto.Common;
-using Svelto.DataStructures;
 using Svelto.DataStructures.Native;
 using Svelto.ECS.DataStructures;
 using Svelto.ECS.Reference;
@@ -12,7 +11,7 @@ namespace Svelto.ECS
     // find the last known EGID from last entity submission.
     public partial class EnginesRoot
     {
-        public struct LocatorMap
+        public struct EntityReferenceMap
         {
             internal EntityReference ClaimReference()
             {
@@ -78,11 +77,12 @@ namespace Svelto.ECS
 
                 // Update reverse map from egid to locator.
                 var groupMap =
-                    _egidToReferenceMap.GetOrCreate(egid.groupID
+                    _egidToReferenceMap.GetOrAdd(egid.groupID
                                                   , () => new SharedSveltoDictionaryNative<uint, EntityReference>(0));
                 groupMap[egid.entityID] = reference;
             }
 
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
             internal void UpdateEntityReference(EGID from, EGID to)
             {
                 var reference = FetchAndRemoveReference(@from);
@@ -90,11 +90,12 @@ namespace Svelto.ECS
                 _entityReferenceMap[reference.index].egid = to;
 
                 var groupMap =
-                    _egidToReferenceMap.GetOrCreate(
+                    _egidToReferenceMap.GetOrAdd(
                         to.groupID, () => new SharedSveltoDictionaryNative<uint, EntityReference>(0));
                 groupMap[to.entityID] = reference;
             }
 
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
             internal void RemoveEntityReference(EGID egid)
             {
                 var reference = FetchAndRemoveReference(@egid);
@@ -108,6 +109,7 @@ namespace Svelto.ECS
                 _nextFreeIndex.Set((int)reference.index);
             }
 
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
             EntityReference FetchAndRemoveReference(EGID @from)
             {
                 var egidToReference = _egidToReferenceMap[@from.groupID];
@@ -117,6 +119,7 @@ namespace Svelto.ECS
                 return reference;
             }
 
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
             internal void RemoveAllGroupReferenceLocators(ExclusiveGroupStruct groupId)
             {
                 if (_egidToReferenceMap.TryGetValue(groupId, out var groupMap) == false)
@@ -125,11 +128,12 @@ namespace Svelto.ECS
                 // We need to traverse all entities in the group and remove the locator using the egid.
                 // RemoveLocator would modify the enumerator so this is why we traverse the dictionary from last to first.
                 foreach (var item in groupMap)
-                    RemoveEntityReference(new EGID(item.Key, groupId));
+                    RemoveEntityReference(new EGID(item.key, groupId));
 
                 _egidToReferenceMap.Remove(groupId);
             }
 
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
             internal void UpdateAllGroupReferenceLocators(ExclusiveGroupStruct fromGroupId, ExclusiveGroupStruct toGroupId)
             {
                 if (_egidToReferenceMap.TryGetValue(fromGroupId, out var groupMap) == false)
@@ -138,7 +142,7 @@ namespace Svelto.ECS
                 // We need to traverse all entities in the group and update the locator using the egid.
                 // UpdateLocator would modify the enumerator so this is why we traverse the dictionary from last to first.
                 foreach (var item in groupMap)
-                    UpdateEntityReference(new EGID(item.Key, fromGroupId), new EGID(item.Key, toGroupId));
+                    UpdateEntityReference(new EGID(item.key, fromGroupId), new EGID(item.key, toGroupId));
 
                 _egidToReferenceMap.Remove(fromGroupId);
             }
@@ -188,8 +192,8 @@ namespace Svelto.ECS
             internal void PreallocateReferenceMaps(ExclusiveGroupStruct groupID, uint size)
             {
                 _egidToReferenceMap
-                   .GetOrCreate(groupID, () => new SharedSveltoDictionaryNative<uint, EntityReference>(size))
-                   .ResizeTo(size);
+                   .GetOrAdd(groupID, () => new SharedSveltoDictionaryNative<uint, EntityReference>(size))
+                   .EnsureCapacity(size);
 
                 _entityReferenceMap.Resize(size);
             }
@@ -211,7 +215,7 @@ namespace Svelto.ECS
                 _entityReferenceMap.Dispose();
 
                 foreach (var element in _egidToReferenceMap)
-                    element.Value.Dispose();
+                    element.value.Dispose();
                 _egidToReferenceMap.Dispose();
             }
 
@@ -222,8 +226,8 @@ namespace Svelto.ECS
                 _egidToReferenceMap;
         }
 
-        internal LocatorMap entityLocator => _entityLocator;
+        EntityReferenceMap entityLocator => _entityLocator;
         
-        LocatorMap          _entityLocator;
+        EntityReferenceMap          _entityLocator;
     }
 }

Core/EntityReference/EntitiesDB.References.cs

@@ -1,5 +1,4 @@
 using System.Runtime.CompilerServices;
-using Svelto.ECS.Reference;
 
 namespace Svelto.ECS
 {
@@ -18,7 +17,7 @@ namespace Svelto.ECS
         }
         
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public EnginesRoot.LocatorMap GetEntityLocator()
+        public EnginesRoot.EntityReferenceMap GetEntityReferenceMap()
         {
             return _entityReferencesMap;
         }

Core/EntityReference/EntityReference.cs

@@ -30,6 +30,8 @@ namespace Svelto.ECS
             return obj1._GID != obj2._GID;
         }
 
+        public override int GetHashCode() { return _GID.GetHashCode(); }
+
         public EntityReference(uint uniqueId) : this(uniqueId, 0) {}
 
         public EntityReference(uint uniqueId, uint version) : this()
@@ -59,7 +61,7 @@ namespace Svelto.ECS
 
             return entitiesDB.GetEGID(this);
         }
-        
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool ToEGID(EntitiesDB entitiesDB, out EGID egid)
         {

Core/EntitySubmissionScheduler.cs

@@ -6,9 +6,9 @@ namespace Svelto.ECS.Schedulers
 
         public abstract void Dispose();
 
-        public abstract bool paused    { get; set; }
-        public          uint iteration { get; protected internal set;  }
-        
+        public bool paused    { get; set; }
+        public uint iteration { get; protected internal set; }
+
         internal bool isRunning;
     }
 }

Core/EntitySubmitOperation.cs

@@ -1,65 +1,10 @@
-using System;
-
-namespace Svelto.ECS
+namespace Svelto.ECS
 {
-#pragma warning disable 660,661
-    struct EntitySubmitOperation
-#pragma warning restore 660,661
-        : IEquatable<EntitySubmitOperation>
-    {
-        public readonly EntitySubmitOperationType type;
-        public readonly IComponentBuilder[]       builders;
-        public readonly EGID                      fromID;
-        public readonly EGID                      toID;
-#if DEBUG && !PROFILE_SVELTO
-        public System.Diagnostics.StackFrame trace;
-#endif
-
-        public EntitySubmitOperation(EntitySubmitOperationType operation, EGID from, EGID to,
-                                     IComponentBuilder[]          builders         = null)
-        {
-            type          = operation;
-            this.builders = builders;
-            fromID        = from;
-            toID          = to;
-#if DEBUG && !PROFILE_SVELTO
-            trace = default;
-#endif
-        }
-
-        public EntitySubmitOperation
-        (EntitySubmitOperationType operation, ExclusiveGroupStruct @group
-       , IComponentBuilder[] descriptorComponentsToBuild):this()
-        {
-            type          = operation;
-            this.builders = descriptorComponentsToBuild;
-            fromID = new EGID(0, group);
-#if DEBUG && !PROFILE_SVELTO
-            trace = default;
-#endif            
-        }
-
-        public static bool operator ==(EntitySubmitOperation obj1, EntitySubmitOperation obj2)
-        {
-            return obj1.Equals(obj2);
-        }
-        
-        public static bool operator !=(EntitySubmitOperation obj1, EntitySubmitOperation obj2)
-        {
-            return obj1.Equals(obj2) == false;
-        }
-
-        public bool Equals(EntitySubmitOperation other)
-        {
-            return type == other.type && fromID == other.fromID && toID == other.toID;
-        }
-    }
-
-    enum EntitySubmitOperationType
+   enum EntitySubmitOperationType
     {
         Swap,
         Remove,
         RemoveGroup,
-        SwapGroup
+        SwapGroup 
     }
 }

Core/Filters/EnginesRoot.Filters.cs

@@ -0,0 +1,160 @@
+using Svelto.DataStructures;
+using Svelto.DataStructures.Native;
+using Svelto.ECS.DataStructures;
+using Svelto.ECS.Internal;
+
+namespace Svelto.ECS
+{
+    public partial class EnginesRoot
+    {
+        void InitFilters()
+        {
+            _transientEntityFilters  = new SharedSveltoDictionaryNative<long, EntityFilterCollection>(0);
+            _persistentEntityFilters = new SharedSveltoDictionaryNative<long, EntityFilterCollection>(0);
+            _indicesOfPersistentFiltersUsedByThisComponent =
+                new SharedSveltoDictionaryNative<NativeRefWrapperType, NativeDynamicArrayCast<int>>(0);
+        }
+
+        void DisposeFilters()
+        {
+            foreach (var filter in _transientEntityFilters)
+            {
+                filter.value.Dispose();
+            }
+
+            foreach (var filter in _persistentEntityFilters)
+            {
+                filter.value.Dispose();
+            }
+
+            foreach (var filter in _indicesOfPersistentFiltersUsedByThisComponent)
+            {
+                filter.value.Dispose();
+            }
+
+            _transientEntityFilters.Dispose();
+            _persistentEntityFilters.Dispose();
+            _indicesOfPersistentFiltersUsedByThisComponent.Dispose();
+        }
+
+        void ClearTransientFilters()
+        {
+            foreach (var filter in _transientEntityFilters)
+            {
+                filter.value.Clear();
+            }
+        }
+
+        void RemoveEntityFromPersistentFilters(FasterList<(uint, string)> entityIDs, ExclusiveGroupStruct fromGroup,
+            RefWrapperType refWrapperType, ITypeSafeDictionary fromDic)
+        {
+            //is there any filter used by this component?
+            if (_indicesOfPersistentFiltersUsedByThisComponent.TryGetValue(new NativeRefWrapperType(refWrapperType),
+                    out NativeDynamicArrayCast<int> listOfFilters))
+            {
+                var numberOfFilters = listOfFilters.count;
+                for (int filterIndex = 0; filterIndex < numberOfFilters; ++filterIndex)
+                {
+                    //we are going to remove multiple entities, this means that the dictionary count would decrease 
+                    //for each entity removed from each filter
+                    //we need to keep a copy to reset to the original count for each filter
+                    var currentLastIndex = (uint)fromDic.count - 1;
+                    var filters          = _persistentEntityFilters.unsafeValues;
+                    var persistentFilter = filters[listOfFilters[filterIndex]]._filtersPerGroup;
+                    
+                    if (persistentFilter.TryGetValue(fromGroup, out var groupFilter))
+                    {
+                        var entitiesCount = entityIDs.count;
+                        
+                        for (int entityIndex = 0; entityIndex < entitiesCount; ++entityIndex)
+                        {
+                            uint fromentityID = entityIDs[entityIndex].Item1;
+                            var  fromIndex    = fromDic.GetIndex(fromentityID);
+
+                            groupFilter.RemoveWithSwapBack(fromentityID, fromIndex, currentLastIndex--);
+                        }
+                    }
+                }
+            }
+        }
+
+        //this method is called by the framework only if listOfFilters.count > 0
+        void SwapEntityBetweenPersistentFilters(FasterList<(uint, uint, string)> fromEntityToEntityIDs,
+            FasterDictionary<uint, uint> beforeSubmissionFromIDs, ITypeSafeDictionary toComponentsDictionary,
+            ExclusiveGroupStruct fromGroup, ExclusiveGroupStruct toGroup, uint fromDictionaryCount,
+            NativeDynamicArrayCast<int> listOfFilters)
+        {
+            DBC.ECS.Check.Require(listOfFilters.count > 0, "why are you calling this with an empty list?");
+            var numberOfFilters = listOfFilters.count;
+
+            /// fromEntityToEntityIDs are the ID of the entities to swap from the from group to the to group.
+            /// for this component type. for each component type, there is only one set of fromEntityToEntityIDs
+            /// per from/to group.
+            /// The complexity of this function is that the ToDictionary is already updated, so the toIndex
+            /// is actually correct and guaranteed to be valid. However the beforeSubmissionFromIDs are the
+            /// indices of the entities in the FromDictionary BEFORE the submission happens, so before the
+            /// entities are actually removed from the dictionary.
+            for (int filterIndex = 0; filterIndex < numberOfFilters; ++filterIndex)
+            {
+                //we are going to remove multiple entities, this means that the dictionary count would decrease 
+                //for each entity removed from each filter
+                //we need to keep a copy to reset to the original count for each filter
+                var currentLastIndex = fromDictionaryCount;
+
+                //if the group has a filter linked:
+                EntityFilterCollection persistentFilter =
+                    _persistentEntityFilters.unsafeValues[listOfFilters[filterIndex]];
+                if (persistentFilter._filtersPerGroup.TryGetValue(fromGroup, out var fromGroupFilter))
+                {
+                    EntityFilterCollection.GroupFilters groupFilterTo = default;
+
+                    foreach (var (fromEntityID, toEntityID, _) in fromEntityToEntityIDs)
+                    {
+                        //if there is an entity, it must be moved to the to filter
+                        if (fromGroupFilter.Exists(fromEntityID) == true)
+                        {
+                            var toIndex = toComponentsDictionary.GetIndex(toEntityID);
+
+                            if (groupFilterTo.isValid == false)
+                                groupFilterTo = persistentFilter.GetGroupFilter(toGroup);
+
+                            groupFilterTo.Add(toEntityID, toIndex);
+                        }
+                    }
+
+                    foreach (var (fromEntityID, _, _) in fromEntityToEntityIDs)
+                    {
+                        //fromIndex is the same of the index in the filter if the entity is in the filter, but
+                        //we need to update the entity index of the last entity swapped from the dictionary even
+                        //in the case when the fromEntity is not present in the filter.
+
+                        uint fromIndex; //index in the from dictionary
+                        if (fromGroupFilter.Exists(fromEntityID))
+                            fromIndex = fromGroupFilter._entityIDToDenseIndex[fromEntityID];
+                        else
+                            fromIndex = beforeSubmissionFromIDs[fromEntityID];
+
+                        //Removing an entity from the dictionary may affect the index of the last entity in the
+                        //values dictionary array, so we need to to update the indices of the affected entities.
+                        //must be outside because from may not be present in the filter, but last index is
+
+                        //for each entity removed from the from group, I have to update it's index in the
+                        //from filter. An entity removed from the DB is always swapped back, which means
+                        //it's current position is taken by the last entity in the dictionary array.
+
+                        //this means that the index of the last entity will change to the index of the
+                        //replaced entity
+
+                        fromGroupFilter.RemoveWithSwapBack(fromEntityID, fromIndex, currentLastIndex--);
+                    }
+                }
+            }
+        }
+
+        internal SharedSveltoDictionaryNative<long, EntityFilterCollection> _transientEntityFilters;
+        internal SharedSveltoDictionaryNative<long, EntityFilterCollection> _persistentEntityFilters;
+
+        internal SharedSveltoDictionaryNative<NativeRefWrapperType, NativeDynamicArrayCast<int>>
+            _indicesOfPersistentFiltersUsedByThisComponent;
+    }
+}

Components/EntityHierarchyComponent.cs.meta → Core/Filters/EnginesRoot.Filters.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: b278b4ce4e7f34f0a2434b7de79b7cbb
+guid: 052b4fc50cbe34f5ab98ee383ee2fb60
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Core/Filters/EntitiesDB.Filters.cs

@@ -0,0 +1,349 @@
+using System;
+using System.Threading;
+using Svelto.Common;
+using Svelto.DataStructures;
+using Svelto.DataStructures.Native;
+using Svelto.ECS.DataStructures;
+
+
+namespace Svelto.ECS
+{
+    public struct FilterContextID
+    {
+        public readonly uint id;
+
+        internal FilterContextID(uint id)
+        {
+            DBC.ECS.Check.Require(id < ushort.MaxValue, "too many types registered, HOW :)");
+
+            this.id = id;
+        }
+    }
+
+    public readonly struct CombinedFilterID
+    {
+        internal readonly long            id;
+        
+        public          FilterContextID contextID => new FilterContextID((uint)((id & 0xFFFF0000) >> 16));
+        public          uint            filterID   => (uint)(id >> 32);
+
+        public CombinedFilterID(int filterID, FilterContextID contextID)
+        {
+            id = (long)filterID << 32 | (uint)contextID.id << 16;
+        }
+
+        public static implicit operator CombinedFilterID((int filterID, FilterContextID contextID) data)
+        {
+            return new CombinedFilterID(data.filterID, data.contextID);
+        }
+    }
+    
+    //this cannot be inside EntitiesDB otherwise it will cause hashing of reference in Burst
+    public class Internal_FilterHelper
+    {
+        //since the user can choose their own filterID, in order to avoid collisions between
+        //filters of the same type, the FilterContext is provided. The type is identified through
+        //TypeCounter
+        public static long CombineFilterIDs<T>(CombinedFilterID combinedFilterID) where T: struct, IEntityComponent
+        {
+            var id = (uint)ComponentID<T>.id.Data;
+
+            var combineFilterIDs = (long)combinedFilterID.id | id;
+
+            return combineFilterIDs;
+        }
+    }
+
+    public partial class EntitiesDB
+    {
+        public SveltoFilters GetFilters()
+        {
+            return new SveltoFilters(_enginesRoot._persistentEntityFilters,
+                _enginesRoot._indicesOfPersistentFiltersUsedByThisComponent, _enginesRoot._transientEntityFilters);
+        }
+
+        /// <summary>
+        /// this whole structure is usable inside DOTS JOBS and BURST
+        /// </summary>
+        public readonly struct SveltoFilters
+        {
+            static readonly SharedStaticWrapper<int, Internal_FilterHelper> uniqueContextID;
+            
+#if UNITY_BURST 
+            [Unity.Burst.BurstDiscard] 
+            //SharedStatic values must be initialized from not burstified code
+#endif            
+            public static FilterContextID GetNewContextID()
+            {
+                return new FilterContextID((uint)Interlocked.Increment(ref uniqueContextID.Data));
+            }
+
+            public SveltoFilters(SharedSveltoDictionaryNative<long, EntityFilterCollection> persistentEntityFilters,
+                SharedSveltoDictionaryNative<NativeRefWrapperType, NativeDynamicArrayCast<int>>
+                    indicesOfPersistentFiltersUsedByThisComponent,
+                SharedSveltoDictionaryNative<long, EntityFilterCollection> transientEntityFilters)
+            {
+                _persistentEntityFilters                       = persistentEntityFilters;
+                _indicesOfPersistentFiltersUsedByThisComponent = indicesOfPersistentFiltersUsedByThisComponent;
+                _transientEntityFilters                        = transientEntityFilters;
+            }
+            
+#if UNITY_BURST
+            public ref EntityFilterCollection GetOrCreatePersistentFilter<T>(int filterID,
+                FilterContextID filterContextId, NativeRefWrapperType typeRef) where T : unmanaged, IEntityComponent
+            {
+                return ref GetOrCreatePersistentFilter<T>(new CombinedFilterID(filterID, filterContextId), typeRef);
+            }
+
+            public ref EntityFilterCollection GetOrCreatePersistentFilter<T>(CombinedFilterID filterID,
+                NativeRefWrapperType typeRef) where T : unmanaged, IEntityComponent
+            {
+                long combineFilterIDs = Internal_FilterHelper.CombineFilterIDs<T>(filterID);
+                
+                if (_persistentEntityFilters.TryFindIndex(combineFilterIDs, out var index) == true)
+                    return ref _persistentEntityFilters.GetDirectValueByRef(index);
+
+                _persistentEntityFilters.Add(combineFilterIDs, new EntityFilterCollection(filterID));
+
+                var lastIndex = _persistentEntityFilters.count - 1;
+
+                if (_indicesOfPersistentFiltersUsedByThisComponent.TryFindIndex(typeRef, out var getIndex) == false)
+                {
+                    var newArray = new NativeDynamicArrayCast<int>(1, Allocator.Persistent);
+                    newArray.Add(lastIndex);
+                    _indicesOfPersistentFiltersUsedByThisComponent.Add(typeRef, newArray);
+                }
+                else
+                {
+                    ref var array = ref _indicesOfPersistentFiltersUsedByThisComponent.GetDirectValueByRef(getIndex);
+                    array.Add(lastIndex);
+                }
+
+                return ref _persistentEntityFilters.GetDirectValueByRef((uint)lastIndex);
+            }
+#endif
+
+            /// <summary>
+            /// Create a persistent filter. Persistent filters are not deleted after each submission,
+            /// however they have a maintenance cost that must be taken into account and will affect
+            /// entities submission performance.
+            /// </summary>
+            /// <typeparam name="T"></typeparam>
+            /// <returns></returns>
+#if UNITY_BURST && UNITY_COLLECTIONS
+            [Unity.Collections.NotBurstCompatible]
+#endif
+            public ref EntityFilterCollection GetOrCreatePersistentFilter<T>(int filterID, FilterContextID filterContextId)
+                where T : unmanaged, IEntityComponent
+            {
+                return ref GetOrCreatePersistentFilter<T>(new CombinedFilterID(filterID, filterContextId));
+            }
+#if UNITY_BURST && UNITY_COLLECTIONS
+            [Unity.Collections.NotBurstCompatible]
+#endif
+            public ref EntityFilterCollection GetOrCreatePersistentFilter<T>(CombinedFilterID filterID)
+                where T : unmanaged, IEntityComponent
+            {
+                long combineFilterIDs = Internal_FilterHelper.CombineFilterIDs<T>(filterID);
+                
+                if (_persistentEntityFilters.TryFindIndex(combineFilterIDs, out var index) == true)
+                    return ref _persistentEntityFilters.GetDirectValueByRef(index);
+
+                var typeRef          = TypeRefWrapper<T>.wrapper;
+                var filterCollection = new EntityFilterCollection(filterID);
+
+                _persistentEntityFilters.Add(combineFilterIDs, filterCollection);
+
+                var lastIndex = _persistentEntityFilters.count - 1;
+
+                _indicesOfPersistentFiltersUsedByThisComponent.GetOrAdd(new NativeRefWrapperType(typeRef),
+                    () => new NativeDynamicArrayCast<int>(1, Svelto.Common.Allocator.Persistent)).Add(lastIndex);
+
+                return ref _persistentEntityFilters.GetDirectValueByRef((uint)lastIndex);
+            }
+
+            public ref EntityFilterCollection GetPersistentFilter<T>(int filterID, FilterContextID filterContextId)
+                where T : unmanaged, IEntityComponent
+            {
+                return ref GetPersistentFilter<T>(new CombinedFilterID(filterID, filterContextId));
+            }
+
+            public ref EntityFilterCollection GetPersistentFilter<T>(CombinedFilterID filterID)
+                where T : unmanaged, IEntityComponent
+            {
+                long combineFilterIDs = Internal_FilterHelper.CombineFilterIDs<T>(filterID);
+                
+                if (_persistentEntityFilters.TryFindIndex(combineFilterIDs, out var index) == true)
+                    return ref _persistentEntityFilters.GetDirectValueByRef(index);
+
+                throw new Exception("filter not found");
+            }
+            
+            public bool TryGetPersistentFilter<T>(CombinedFilterID combinedFilterID, out EntityFilterCollection entityCollection) 
+                where T : unmanaged, IEntityComponent
+            {
+                long combineFilterIDs = Internal_FilterHelper.CombineFilterIDs<T>(combinedFilterID);
+                
+                if (_persistentEntityFilters.TryFindIndex(combineFilterIDs, out var index) == true)
+                {
+                    entityCollection = _persistentEntityFilters.GetDirectValueByRef(index);
+                    return true;
+                }
+
+                entityCollection = default;
+                return false;
+            }
+
+            public EntityFilterCollectionsEnumerator GetPersistentFilters<T>() where T : unmanaged, IEntityComponent
+            {
+                if (_indicesOfPersistentFiltersUsedByThisComponent.TryFindIndex(
+                        new NativeRefWrapperType(new RefWrapperType(typeof(T))), out var index) == true)
+                    return new EntityFilterCollectionsEnumerator(
+                        _indicesOfPersistentFiltersUsedByThisComponent.GetDirectValueByRef(index),
+                        _persistentEntityFilters);
+
+                throw new Exception($"no filters associated with the type {TypeCache<T>.name}");
+            }
+            
+            public EntityFilterCollectionsWithContextEnumerator GetPersistentFilters<T>(FilterContextID filterContextId)
+            {
+                if (_indicesOfPersistentFiltersUsedByThisComponent.TryFindIndex(
+                        new NativeRefWrapperType(new RefWrapperType(typeof(T))), out var index) == true)
+                    return new EntityFilterCollectionsWithContextEnumerator(
+                        _indicesOfPersistentFiltersUsedByThisComponent.GetDirectValueByRef(index),
+                        _persistentEntityFilters, filterContextId);
+
+                throw new Exception($"no filters associated with the type {TypeCache<T>.name}");
+            }
+            
+            public bool TryGetPersistentFilters<T>(FilterContextID filterContextId, out EntityFilterCollectionsWithContextEnumerator enumerator)
+            {
+                if (_indicesOfPersistentFiltersUsedByThisComponent.TryFindIndex(
+                        new NativeRefWrapperType(new RefWrapperType(typeof(T))), out var index) == true)
+                {
+                    enumerator = new EntityFilterCollectionsWithContextEnumerator(
+                        _indicesOfPersistentFiltersUsedByThisComponent.GetDirectValueByRef(index),
+                        _persistentEntityFilters, filterContextId);
+
+                    return true;
+                }
+
+                enumerator = default;
+                return false;
+            }
+
+            public struct EntityFilterCollectionsEnumerator
+            {
+                public EntityFilterCollectionsEnumerator(NativeDynamicArrayCast<int> getDirectValueByRef,
+                    SharedSveltoDictionaryNative<long, EntityFilterCollection> sharedSveltoDictionaryNative) : this()
+                {
+                    _getDirectValueByRef          = getDirectValueByRef;
+                    _sharedSveltoDictionaryNative = sharedSveltoDictionaryNative;
+                }
+
+                public EntityFilterCollectionsEnumerator GetEnumerator()
+                {
+                    return this;
+                }
+
+                public bool MoveNext()
+                {
+                    if (_currentIndex < _getDirectValueByRef.count)
+                    {
+                        _currentIndex++;
+                        
+                        return true;
+                    }
+
+                    return false;
+                }
+
+                public ref EntityFilterCollection Current =>
+                    ref _sharedSveltoDictionaryNative.GetDirectValueByRef((uint)_currentIndex - 1);
+
+                readonly NativeDynamicArrayCast<int>                                _getDirectValueByRef;
+                readonly SharedSveltoDictionaryNative<long, EntityFilterCollection> _sharedSveltoDictionaryNative;
+                int                                                                 _currentIndex;
+            }
+            
+            public struct EntityFilterCollectionsWithContextEnumerator
+            {
+                public EntityFilterCollectionsWithContextEnumerator(NativeDynamicArrayCast<int> getDirectValueByRef,
+                    SharedSveltoDictionaryNative<long, EntityFilterCollection> sharedSveltoDictionaryNative,
+                    FilterContextID filterContextId) : this()
+                {
+                    _getDirectValueByRef          = getDirectValueByRef;
+                    _sharedSveltoDictionaryNative = sharedSveltoDictionaryNative;
+                    _filterContextId         = filterContextId;
+                }
+            
+                public EntityFilterCollectionsWithContextEnumerator GetEnumerator()
+                {
+                    return this;
+                }
+            
+                public bool MoveNext()
+                {
+                    while (_currentIndex++ < _getDirectValueByRef.count &&
+                           _sharedSveltoDictionaryNative.GetDirectValueByRef((uint)_currentIndex - 1).combinedFilterID
+                              .contextID.id != _filterContextId.id) ;
+                            
+                    if (_currentIndex - 1 < _getDirectValueByRef.count) 
+                        return true;
+                    
+                    return false;
+                }
+            
+                public ref EntityFilterCollection Current =>
+                    ref _sharedSveltoDictionaryNative.GetDirectValueByRef((uint)_currentIndex - 1);
+            
+                readonly NativeDynamicArrayCast<int>                                _getDirectValueByRef;
+                readonly SharedSveltoDictionaryNative<long, EntityFilterCollection> _sharedSveltoDictionaryNative;
+                readonly FilterContextID                                            _filterContextId;
+                int                                                                 _currentIndex;
+            }
+
+            /// <summary>
+            /// Creates a transient filter. Transient filters are deleted after each submission
+            /// </summary>
+            /// <typeparam name="T"></typeparam>
+            /// <returns></returns>
+            public ref EntityFilterCollection GetOrCreateTransientFilter<T>(CombinedFilterID filterID)
+                where T : unmanaged, IEntityComponent
+            {
+                var combineFilterIDs = Internal_FilterHelper.CombineFilterIDs<T>(filterID);
+
+                if (_transientEntityFilters.TryFindIndex(combineFilterIDs, out var index))
+                    return ref _transientEntityFilters.GetDirectValueByRef(index);
+
+                var filterCollection = new EntityFilterCollection(filterID);
+
+                _transientEntityFilters.Add(combineFilterIDs, filterCollection);
+
+                return ref _transientEntityFilters.GetDirectValueByRef((uint)(_transientEntityFilters.count - 1));
+            }
+
+            public bool TryGetTransientFilter<T>(CombinedFilterID filterID, out EntityFilterCollection entityCollection)
+                where T : unmanaged, IEntityComponent
+            {
+                var combineFilterIDs = Internal_FilterHelper.CombineFilterIDs<T>(filterID);
+
+                if (_transientEntityFilters.TryFindIndex(combineFilterIDs, out var index))
+                {
+                    entityCollection = _transientEntityFilters.GetDirectValueByRef(index);
+                    return true;
+                }
+
+                entityCollection = default;
+                return false;
+            }
+
+            readonly SharedSveltoDictionaryNative<long, EntityFilterCollection> _persistentEntityFilters;
+
+            readonly SharedSveltoDictionaryNative<NativeRefWrapperType, NativeDynamicArrayCast<int>>
+                _indicesOfPersistentFiltersUsedByThisComponent;
+
+            readonly SharedSveltoDictionaryNative<long, EntityFilterCollection> _transientEntityFilters;
+        }
+    }
+}

Core/Filters/EntitiesDB.GroupFilters.cs.meta → Core/Filters/EntitiesDB.Filters.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 5a8f3a6e101838a88604fe66d87cf9d6
+guid: 65dbc18996da37ff8a18d67e63b37f73
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Core/Filters/EntitiesDB.GroupFilters.cs → Core/Filters/EntitiesDB.LegacyFilters.cs

@@ -1,10 +1,12 @@
+using DBC.ECS;
 using Svelto.DataStructures;
 using Svelto.DataStructures.Native;
 
 namespace Svelto.ECS
 {
     /// <summary>
-    ///     This feature must be eventually tied to the new ExclusiveGroup that won't allow the use of custom EntitiesID
+    ///     This feature must be eventually tied to the new ExclusiveGroup that won't allow the use of
+    /// custom EntitiesID
     ///     The filters could be updated when entities buffer changes during the submission, while now this process
     ///     is completely manual.
     ///     Making this working properly is not in my priorities right now, as I will need to add the new group type
@@ -12,15 +14,24 @@ namespace Svelto.ECS
     /// </summary>
     public partial class EntitiesDB
     {
-        public readonly struct Filters
+        FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, LegacyGroupFilters>> _filters =>
+            _enginesRoot._groupFilters;
+
+        public LegacyFilters GetLegacyFilters()
         {
-            public Filters
-                (FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, GroupFilters>> filters)
+            return new LegacyFilters(_filters);
+        }
+
+        public readonly struct LegacyFilters
+        {
+            public LegacyFilters(
+                FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, LegacyGroupFilters>>
+                    filtersLegacy)
             {
-                _filters = filters;
+                _filtersLegacy = filtersLegacy;
             }
 
-            public ref FilterGroup CreateOrGetFilterForGroup<T>(int filterID, ExclusiveGroupStruct groupID)
+            public ref LegacyFilterGroup CreateOrGetFilterForGroup<T>(int filterID, ExclusiveGroupStruct groupID)
                 where T : struct, IEntityComponent
             {
                 var refWrapper = TypeRefWrapper<T>.wrapper;
@@ -28,21 +39,9 @@ namespace Svelto.ECS
                 return ref CreateOrGetFilterForGroup(filterID, groupID, refWrapper);
             }
 
-            internal ref FilterGroup CreateOrGetFilterForGroup
-                (int filterID, ExclusiveGroupStruct groupID, RefWrapperType refWrapper)
-            {
-                var fasterDictionary =
-                    _filters.GetOrCreate(refWrapper, () => new FasterDictionary<ExclusiveGroupStruct, GroupFilters>());
-
-                GroupFilters filters = fasterDictionary.GetOrCreate(
-                    groupID, () => new GroupFilters(new SharedSveltoDictionaryNative<int, FilterGroup>(0), groupID));
-
-                return ref filters.CreateOrGetFilter(filterID);
-            }
-
             public bool HasFiltersForGroup<T>(ExclusiveGroupStruct groupID) where T : struct, IEntityComponent
             {
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
                     return false;
 
                 return fasterDictionary.ContainsKey(groupID);
@@ -51,7 +50,7 @@ namespace Svelto.ECS
             public bool HasFilterForGroup<T>(int filterID, ExclusiveGroupStruct groupID)
                 where T : struct, IEntityComponent
             {
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
                     return false;
 
                 if (fasterDictionary.TryGetValue(groupID, out var result))
@@ -60,79 +59,77 @@ namespace Svelto.ECS
                 return false;
             }
 
-            public ref GroupFilters CreateOrGetFiltersForGroup<T>(ExclusiveGroupStruct groupID)
+            public ref LegacyGroupFilters CreateOrGetFiltersForGroup<T>(ExclusiveGroupStruct groupID)
                 where T : struct, IEntityComponent
             {
-                var fasterDictionary = _filters.GetOrCreate(TypeRefWrapper<T>.wrapper
-                                                          , () =>
-                                                                new FasterDictionary<ExclusiveGroupStruct,
-                                                                    GroupFilters>());
+                var fasterDictionary = _filtersLegacy.GetOrAdd(TypeRefWrapper<T>.wrapper,
+                    () => new FasterDictionary<ExclusiveGroupStruct, LegacyGroupFilters>());
 
-                return ref fasterDictionary.GetOrCreate(
-                    groupID, () => new GroupFilters(new SharedSveltoDictionaryNative<int, FilterGroup>(0), groupID));
+                return ref fasterDictionary.GetOrAdd(groupID,
+                    () => new LegacyGroupFilters(new SharedSveltoDictionaryNative<int, LegacyFilterGroup>(0), groupID));
             }
 
-            public ref GroupFilters GetFiltersForGroup<T>(ExclusiveGroupStruct groupID)
+            public ref LegacyGroupFilters GetFiltersForGroup<T>(ExclusiveGroupStruct groupID)
                 where T : struct, IEntityComponent
             {
 #if DEBUG && !PROFILE_SVELTO
-                if (_filters.ContainsKey(TypeRefWrapper<T>.wrapper) == false)
+                if (_filtersLegacy.ContainsKey(TypeRefWrapper<T>.wrapper) == false)
                     throw new ECSException($"trying to fetch not existing filters, type {typeof(T)}");
-                if (_filters[TypeRefWrapper<T>.wrapper].ContainsKey(groupID) == false)
+                if (_filtersLegacy[TypeRefWrapper<T>.wrapper].ContainsKey(groupID) == false)
                     throw new ECSException(
                         $"trying to fetch not existing filters, type {typeof(T)} group {groupID.ToName()}");
 #endif
 
-                return ref _filters[TypeRefWrapper<T>.wrapper].GetValueByRef(groupID);
+                return ref _filtersLegacy[TypeRefWrapper<T>.wrapper].GetValueByRef(groupID);
             }
 
-            public ref FilterGroup GetFilterForGroup<T>(int filterId, ExclusiveGroupStruct groupID)
+            public ref LegacyFilterGroup GetFilterForGroup<T>(int filterId, ExclusiveGroupStruct groupID)
                 where T : struct, IEntityComponent
             {
 #if DEBUG && !PROFILE_SVELTO
-                if (_filters.ContainsKey(TypeRefWrapper<T>.wrapper) == false)
+                if (_filtersLegacy.ContainsKey(TypeRefWrapper<T>.wrapper) == false)
                     throw new ECSException($"trying to fetch not existing filters, type {typeof(T)}");
-                if (_filters[TypeRefWrapper<T>.wrapper].ContainsKey(groupID) == false)
+                if (_filtersLegacy[TypeRefWrapper<T>.wrapper].ContainsKey(groupID) == false)
                     throw new ECSException(
                         $"trying to fetch not existing filters, type {typeof(T)} group {groupID.ToName()}");
 #endif
-                return ref _filters[TypeRefWrapper<T>.wrapper][groupID].GetFilter(filterId);
+                return ref _filtersLegacy[TypeRefWrapper<T>.wrapper][groupID].GetFilter(filterId);
             }
 
-            public bool TryGetFilterForGroup<T>(int filterId, ExclusiveGroupStruct groupID, out FilterGroup groupFilter)
-                where T : struct, IEntityComponent
+            public bool TryGetFilterForGroup<T>(int filterId, ExclusiveGroupStruct groupID,
+                out LegacyFilterGroup groupLegacyFilter) where T : struct, IEntityComponent
             {
-                groupFilter = default;
+                groupLegacyFilter = default;
 
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
                     return false;
 
                 if (fasterDictionary.TryGetValue(groupID, out var groupFilters) == false)
                     return false;
 
-                if (groupFilters.TryGetFilter(filterId, out groupFilter) == false)
+                if (groupFilters.TryGetFilter(filterId, out groupLegacyFilter) == false)
                     return false;
 
                 return true;
             }
 
-            public bool TryGetFiltersForGroup<T>(ExclusiveGroupStruct groupID, out GroupFilters groupFilters)
-                where T : struct, IEntityComponent
+            public bool TryGetFiltersForGroup<T>(ExclusiveGroupStruct groupID,
+                out LegacyGroupFilters legacyGroupFilters) where T : struct, IEntityComponent
             {
-                groupFilters = default;
+                legacyGroupFilters = default;
 
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == false)
                     return false;
 
-                return fasterDictionary.TryGetValue(groupID, out groupFilters);
+                return fasterDictionary.TryGetValue(groupID, out legacyGroupFilters);
             }
 
             public void ClearFilter<T>(int filterID, ExclusiveGroupStruct exclusiveGroupStruct)
             {
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == true)
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary))
                 {
-                    DBC.ECS.Check.Require(fasterDictionary.ContainsKey(exclusiveGroupStruct)
-                                        , $"trying to clear filter not present in group {exclusiveGroupStruct}");
+                    Check.Require(fasterDictionary.ContainsKey(exclusiveGroupStruct),
+                        $"trying to clear filter not present in group {exclusiveGroupStruct}");
 
                     fasterDictionary[exclusiveGroupStruct].ClearFilter(filterID);
                 }
@@ -140,16 +137,14 @@ namespace Svelto.ECS
 
             public void ClearFilters<T>(int filterID)
             {
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == true)
-                {
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary))
                     foreach (var filtersPerGroup in fasterDictionary)
-                        filtersPerGroup.Value.ClearFilter(filterID);
-                }
+                        filtersPerGroup.value.ClearFilter(filterID);
             }
 
             public void DisposeFilters<T>(ExclusiveGroupStruct exclusiveGroupStruct)
             {
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == true)
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary))
                 {
                     fasterDictionary[exclusiveGroupStruct].DisposeFilters();
                     fasterDictionary.Remove(exclusiveGroupStruct);
@@ -158,29 +153,23 @@ namespace Svelto.ECS
 
             public void DisposeFilters<T>()
             {
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == true)
-                {
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary))
                     foreach (var filtersPerGroup in fasterDictionary)
-                        filtersPerGroup.Value.DisposeFilters();
-                }
+                        filtersPerGroup.value.DisposeFilters();
 
-                _filters.Remove(TypeRefWrapper<T>.wrapper);
+                _filtersLegacy.Remove(TypeRefWrapper<T>.wrapper);
             }
 
-            public void DisposeFilterForGroup<T>(int resetFilterID, ExclusiveGroupStruct @group)
+            public void DisposeFilterForGroup<T>(int resetFilterID, ExclusiveGroupStruct group)
             {
-                if (_filters.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary) == true)
-                {
-                    fasterDictionary[group].DisposeFilter(resetFilterID);
-                }
+                if (_filtersLegacy.TryGetValue(TypeRefWrapper<T>.wrapper, out var fasterDictionary))
+                    fasterDictionary[@group].DisposeFilter(resetFilterID);
             }
 
             public bool TryRemoveEntityFromFilter<T>(int filtersID, EGID egid) where T : struct, IEntityComponent
             {
                 if (TryGetFilterForGroup<T>(filtersID, egid.groupID, out var filter))
-                {
                     return filter.TryRemove(egid.entityID);
-                }
 
                 return false;
             }
@@ -200,13 +189,22 @@ namespace Svelto.ECS
                 return filter.Add(egid.entityID, mapper);
             }
 
-            readonly FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, GroupFilters>> _filters;
-        }
+            internal ref LegacyFilterGroup CreateOrGetFilterForGroup(int filterID, ExclusiveGroupStruct groupID,
+                RefWrapperType refWrapper)
+            {
+                var fasterDictionary = _filtersLegacy.GetOrAdd(refWrapper,
+                    () => new FasterDictionary<ExclusiveGroupStruct, LegacyGroupFilters>());
 
-        public Filters GetFilters() { return new Filters(_filters); }
+                var filters = fasterDictionary.GetOrAdd(groupID,
+                    (ref ExclusiveGroupStruct gid) =>
+                        new LegacyGroupFilters(new SharedSveltoDictionaryNative<int, LegacyFilterGroup>(0), gid),
+                    ref groupID);
 
+                return ref filters.CreateOrGetFilter(filterID);
+            }
 
-        FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, GroupFilters>> _filters =>
-            _enginesRoot._groupFilters;
+            readonly FasterDictionary<RefWrapperType, FasterDictionary<ExclusiveGroupStruct, LegacyGroupFilters>>
+                _filtersLegacy;
+        }
     }
 }

Components/EntityReferenceComponent.cs.meta → Core/Filters/EntitiesDB.LegacyFilters.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 845c1c4af4ca3dd598319878015a84f3
+guid: 3f1a93ced19b393997928e037fcd007a
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Core/Filters/EntityFilterCollection.cs

@@ -0,0 +1,198 @@
+using System.Runtime.CompilerServices;
+using Svelto.Common;
+using Svelto.DataStructures.Native;
+using Svelto.ECS.Native;
+
+namespace Svelto.ECS
+{
+    public readonly struct EntityFilterCollection
+    {
+        internal EntityFilterCollection(CombinedFilterID combinedFilterId,
+            Allocator allocatorStrategy = Allocator.Persistent)
+        {
+            _filtersPerGroup =
+                SharedSveltoDictionaryNative<ExclusiveGroupStruct, GroupFilters>.Create(allocatorStrategy);
+
+            combinedFilterID = combinedFilterId;
+        }
+
+        public CombinedFilterID combinedFilterID { get; }
+        
+        public EntityFilterIterator GetEnumerator()  => new EntityFilterIterator(this);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool Add<T>(EGID egid, NativeEGIDMapper<T> mmap) where T : unmanaged, IEntityComponent
+        {
+            DBC.ECS.Check.Require(mmap.groupID == egid.groupID, "not compatible NativeEgidMapper used");
+
+            return Add(egid, mmap.GetIndex(egid.entityID));
+        }
+
+        public bool Add<T>(EGID egid, NativeEGIDMultiMapper<T> mmap) where T : unmanaged, IEntityComponent
+        {
+            return Add(egid, mmap.GetIndex(egid));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool Add(EGID egid, uint toIndex)
+        {
+            return GetGroupFilter(egid.groupID).Add(egid.entityID, toIndex);
+        }
+        
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Add(uint entityID, ExclusiveGroupStruct groupId, uint index)
+        {
+            Add(new EGID(entityID, groupId), index);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Remove(EGID egid)
+        {
+            _filtersPerGroup[egid.groupID].Remove(egid.entityID);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool Exists(EGID egid)
+        {
+            return GetGroupFilter(egid.groupID).Exists(egid.entityID);
+        }
+        
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public GroupFilters GetGroupFilter(ExclusiveGroupStruct group)
+        {
+            if (_filtersPerGroup.TryGetValue(group, out var groupFilter) == false)
+            {
+                groupFilter = new GroupFilters(group);
+                _filtersPerGroup.Add(group, groupFilter);
+            }
+
+            return groupFilter;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Clear()
+        {
+            var filterSets = _filtersPerGroup.GetValues(out var count);
+            for (var i = 0; i < count; i++)
+            {
+                filterSets[i].Clear();
+            }
+        }
+
+        internal int groupCount => _filtersPerGroup.count;
+        
+        public void ComputeFinalCount(out int count)
+        {
+            count = 0;
+            
+            for (int i = 0; i < _filtersPerGroup.count; i++)
+            {
+                count += (int)GetGroup(i).count;
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal GroupFilters GetGroup(int indexGroup)
+        {
+            DBC.ECS.Check.Require(indexGroup < _filtersPerGroup.count);
+            return _filtersPerGroup.GetValues(out _)[indexGroup];
+        }
+
+        public void Dispose()
+        {
+            var filterSets = _filtersPerGroup.GetValues(out var count);
+            for (var i = 0; i < count; i++)
+            {
+                filterSets[i].Dispose();
+            }
+
+            _filtersPerGroup.Dispose();
+        }
+
+        internal readonly SharedSveltoDictionaryNative<ExclusiveGroupStruct, GroupFilters> _filtersPerGroup;
+
+        public struct GroupFilters
+        {
+            internal GroupFilters(ExclusiveGroupStruct group) : this()
+            {
+                _entityIDToDenseIndex = new SharedSveltoDictionaryNative<uint, uint>(1);
+                _indexToEntityId      = new SharedSveltoDictionaryNative<uint, uint>(1);
+                _group                = group;
+            }
+
+            public bool Add(uint entityId, uint entityIndex)
+            {
+                //TODO: when sentinels are finished, we need to add AsWriter here
+                if (_entityIDToDenseIndex.TryAdd(entityId, entityIndex, out _))
+                {
+                    _indexToEntityId[entityIndex] = entityId;
+                    return true;
+                }
+
+                return false;
+            }
+
+            public bool Exists(uint entityId) => _entityIDToDenseIndex.ContainsKey(entityId);
+
+            public void Remove(uint entityId)
+            {
+                _indexToEntityId.Remove(_entityIDToDenseIndex[entityId]);
+                _entityIDToDenseIndex.Remove(entityId);
+            }
+
+            public EntityFilterIndices indices
+            {
+                get
+                {
+                    var values = _entityIDToDenseIndex.GetValues(out var count);
+                    return new EntityFilterIndices(values, count);
+                }
+            }
+
+            public int count   => _entityIDToDenseIndex.count;
+            public bool isValid => _entityIDToDenseIndex.isValid;
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            internal void RemoveWithSwapBack(uint entityId, uint entityIndex, uint lastIndex)
+            {
+                // Check if the last index is part of the filter as an entity, in that case
+                //we need to update the filter
+                if (entityIndex != lastIndex && _indexToEntityId.TryGetValue(lastIndex, out var lastEntityID))
+                {
+                    _entityIDToDenseIndex[lastEntityID] = entityIndex;
+                    _indexToEntityId[entityIndex]       = lastEntityID;
+
+                    _indexToEntityId.Remove(lastIndex);
+                }
+                else
+                {
+                    // We don't need to check if the entityIndex is a part of the dictionary.
+                    // The Remove function will check for us.
+                    _indexToEntityId.Remove(entityIndex);
+                }
+
+                // We don't need to check if the entityID is part of the dictionary.
+                // The Remove function will check for us.
+                _entityIDToDenseIndex.Remove(entityId);
+            }
+
+            internal void Clear()
+            {
+                _indexToEntityId.FastClear();
+                _entityIDToDenseIndex.FastClear();
+            }
+
+            internal void Dispose()
+            {
+                _entityIDToDenseIndex.Dispose();
+                _indexToEntityId.Dispose();
+            }
+
+            internal ExclusiveGroupStruct group => _group;
+
+            SharedSveltoDictionaryNative<uint, uint>          _indexToEntityId;
+            internal SharedSveltoDictionaryNative<uint, uint> _entityIDToDenseIndex;
+            readonly ExclusiveGroupStruct                     _group;
+        }
+    }
+}

Core/Filters/EntityFilterCollection.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: c399dabb17763f8ca9109916f27d8ea4
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/EntityFilterID.cs

@@ -0,0 +1,32 @@
+using System;
+using Svelto.DataStructures;
+
+namespace Svelto.ECS
+{
+    // This is just an opaque struct to identify filter collections.
+    public struct EntityFilterID : IEquatable<EntityFilterID>
+    {
+        internal EntityFilterID(uint filterID, RefWrapperType componentType)
+        {
+            _filterID = filterID;
+            _componentType = componentType;
+            _hashCode = (int)filterID + (int)filterID ^ componentType.GetHashCode();
+        }
+
+        public bool Equals(EntityFilterID other)
+        {
+            return _filterID == other._filterID && _componentType.Equals(other._componentType);
+        }
+
+        public override bool Equals(object obj)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override int GetHashCode() => _hashCode;
+
+        readonly uint           _filterID;
+        readonly RefWrapperType _componentType;
+        readonly int            _hashCode;
+    }
+}

Core/Filters/EntityFilterID.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d61208b8891d3e6e8265ec8d46b45080
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/EntityFilterIndices.cs

@@ -0,0 +1,47 @@
+using System.Runtime.CompilerServices;
+using System.Threading;
+using Svelto.DataStructures;
+
+namespace Svelto.ECS
+{
+    public struct EntityFilterIndices
+    {
+        public EntityFilterIndices(NB<uint> indices, uint count)
+        {
+            _indices = indices;
+            _count   = count;
+            _index   = 0;
+        }
+
+        public uint count
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _count;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public uint Get(uint index) => _indices[index];
+
+        public uint this[uint index]
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _indices[index];
+        }
+
+        public uint this[int index]
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _indices[index];
+        }
+        
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public uint Next()
+        {
+            return _indices[Interlocked.Increment(ref _index) - 1];
+        }
+
+        readonly NB<uint> _indices;
+        readonly uint     _count;
+        int               _index;
+    }
+}

Core/Filters/EntityFilterIndices.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 73be3ee2f042368c93f1edf214b512d8
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/EntityFilterIterator.cs

@@ -0,0 +1,51 @@
+namespace Svelto.ECS
+{
+    public ref struct EntityFilterIterator
+    {
+        internal EntityFilterIterator(EntityFilterCollection  filter)
+        {
+            _filter     = filter;
+            _indexGroup = -1;
+            _current    = default;
+        }
+
+        public bool MoveNext()
+        {
+            while (++_indexGroup < _filter.groupCount)
+            {
+                _current = _filter.GetGroup(_indexGroup);
+
+                if (_current.count > 0) break;
+            }
+
+            return _indexGroup < _filter.groupCount;
+        }
+
+        public void Reset()
+        {
+            _indexGroup = -1;
+        }
+
+        public RefCurrent Current => new RefCurrent(_current);
+
+        int                                 _indexGroup;
+        readonly EntityFilterCollection     _filter;
+        EntityFilterCollection.GroupFilters _current;
+
+        public readonly ref struct RefCurrent
+        {
+            internal RefCurrent(EntityFilterCollection.GroupFilters filter)
+            {
+                _filter = filter;
+            }
+
+            public void Deconstruct(out EntityFilterIndices indices, out ExclusiveGroupStruct group)
+            {
+                indices = _filter.indices;
+                group   = _filter.group;
+            }
+
+            readonly EntityFilterCollection.GroupFilters _filter;
+        }
+    }
+}

Core/Filters/EntityFilterIterator.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: da6d607b764d347082e7b91f764af45d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/FilterGroup.cs.meta

@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 2f13dc2395a2359fa5acac05ebaafc41
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 

Core/Filters/FilteredIndices.cs.meta

@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 8d699c30169d326ba534c80d1ab5c504
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 

Core/Filters/GroupFilters.cs

@@ -1,104 +0,0 @@
-using Svelto.DataStructures;
-using Svelto.DataStructures.Native;
-
-namespace Svelto.ECS
-{
-    public struct GroupFilters
-    {
-        internal GroupFilters(SharedSveltoDictionaryNative<int, FilterGroup> filters, ExclusiveGroupStruct group)
-        {
-            this.filters = filters;
-            _group       = @group;
-        }
-
-        public ref FilterGroup GetFilter(int filterIndex)
-        {
-#if DEBUG && !PROFILE_SVELTO
-            if (filters.isValid == false)
-                throw new ECSException($"trying to fetch not existing filters {filterIndex} group {_group.ToName()}");
-            if (filters.ContainsKey(filterIndex) == false)
-                throw new ECSException($"trying to fetch not existing filters {filterIndex} group {_group.ToName()}");
-#endif
-            return ref filters.GetValueByRef(filterIndex);
-        }
-
-        public bool HasFilter(int filterIndex) { return filters.ContainsKey(filterIndex); }
-
-        public void ClearFilter(int filterIndex)
-        {
-            if (filters.TryFindIndex(filterIndex, out var index))
-                filters.GetValues(out _)[index].Clear();
-        }
-
-        public void ClearFilters()
-        {
-            foreach (var filter in filters)
-                filter.Value.Clear();
-        }
-
-        public bool TryGetFilter(int filterIndex, out FilterGroup filter)
-        {
-            return filters.TryGetValue(filterIndex, out filter);
-        }
-
-        public SveltoDictionaryKeyValueEnumerator<int, FilterGroup, NativeStrategy<SveltoDictionaryNode<int>>, NativeStrategy<FilterGroup>
-          , NativeStrategy<int>> GetEnumerator()
-        {
-            return filters.GetEnumerator();
-        }
-        
-        //Note the following methods are internal because I was pondering the idea to be able to return
-        //the list of GroupFilters linked to a specific filter ID. However this would mean to be able to
-        //maintain a revers map which at this moment seems too much and also would need the following
-        //method to be for ever internal (at this point in time I am not sure it's a good idea)
-        internal void DisposeFilter(int filterIndex)
-        {
-            if (filters.TryFindIndex(filterIndex, out var index))
-            {
-                ref var filterGroup = ref filters.GetValues(out _)[index];
-                
-                filterGroup.Dispose();
-
-                filters.Remove(filterIndex);
-            }
-        }
-
-        internal void DisposeFilters()
-        {
-            //must release the native buffers!
-            foreach (var filter in filters)
-                filter.Value.Dispose();
-
-            filters.FastClear();
-        }
-
-        internal ref FilterGroup CreateOrGetFilter(int filterID)
-        {
-            if (filters.TryFindIndex(filterID, out var index) == false)
-            {
-                var orGetFilterForGroup = new FilterGroup(_group, filterID);
-
-                filters[filterID] = orGetFilterForGroup;
-
-                return ref filters.GetValueByRef(filterID);
-            }
-
-            return ref filters.GetValues(out _)[index];
-        }
-
-        internal void Dispose()
-        {
-            foreach (var filter in filters)
-            {
-                filter.Value.Dispose();
-            }
-
-            filters.Dispose();
-        }
-
-        readonly ExclusiveGroupStruct _group;
-
-        //filterID, filter
-        SharedSveltoDictionaryNative<int, FilterGroup> filters;
-    }
-}

Core/Filters/GroupFilters.cs.meta

@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 7f1fcb3722433ba98f34384a1553a661
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 

Core/Filters/FilterGroup.cs → Core/Filters/LegacyFilterGroup.cs

@@ -1,6 +1,5 @@
 using System.Runtime.CompilerServices;
 using Svelto.Common;
-using Svelto.DataStructures;
 using Svelto.DataStructures.Native;
 using Svelto.ECS.DataStructures;
 
@@ -16,9 +15,9 @@ namespace Svelto.ECS
     ///     sparse[0] = position in the dense list of the entity 0
     ///     dense[index] = entity ID but also index in the sparse list of the same entity ID
     /// </summary>
-    public struct FilterGroup
+    public struct LegacyFilterGroup
     {
-        internal FilterGroup(ExclusiveGroupStruct exclusiveGroupStruct, int ID)
+        internal LegacyFilterGroup(ExclusiveGroupStruct exclusiveGroupStruct, int ID)
         {
             _denseListOfIndicesToEntityComponentArray =
                 new NativeDynamicArrayCast<uint>(NativeDynamicArray.Alloc<uint>(Allocator.Persistent));
@@ -33,7 +32,7 @@ namespace Svelto.ECS
         /// <summary>
         /// Todo: how to detect if the indices are still pointing to valid entities?
         /// </summary>
-        public FilteredIndices filteredIndices => new FilteredIndices(_denseListOfIndicesToEntityComponentArray);
+        public LegacyFilteredIndices filteredIndices => new LegacyFilteredIndices(_denseListOfIndicesToEntityComponentArray);
 
         public bool Add<N>(uint entityID, N mapper)  where N:IEGIDMapper
         {
@@ -85,11 +84,11 @@ namespace Svelto.ECS
         /// Filters were initially designed to be used for tagging operations within submissions of entities.
         /// They were designed as a fast tagging mechanism to be used within the submission frame. However I then
         /// extended it, but the extension includes a drawback:
-        ///If filters are not in sync with the operations of remove and swap, filters may end up pointing to
-        ///invalid indices. I need to put in place a way to be able to recognised an invalid filter.
-        ///This is currently a disadvantage of the filters. The filters are not updated by the framework
-        ///but they must be updated by the user.
-        ///When to use this method: Add and Removed should be used to add and remove entities in the filters. This is
+        /// If filters are not in sync with the operations of remove and swap, filters may end up pointing to
+        /// invalid indices. I need to put in place a way to be able to recognised an invalid filter.
+        /// This is currently a disadvantage of the filters. The filters are not updated by the framework
+        /// but they must be updated by the user.
+        /// When to use this method: Add and Removed should be used to add and remove entities in the filters. This is
         /// valid as long as no structural changes happen in the group of entities involved.
         /// IF structural changes happen, the indices stored in the filters won't be valid anymore as they will possibly
         /// point to entities that were not the original ones. On structural changes
@@ -107,11 +106,11 @@ namespace Svelto.ECS
             _reverseEIDs.Clear();
 
             foreach (var value in _indexOfEntityInDenseList)
-                if (mapper.FindIndex(value.Key, out var indexOfEntityInBufferComponent) == true)
+                if (mapper.FindIndex(value.key, out var indexOfEntityInBufferComponent) == true)
                 {
                     _denseListOfIndicesToEntityComponentArray.Add(indexOfEntityInBufferComponent);
                     var lastIndex = (uint) (_denseListOfIndicesToEntityComponentArray.Count() - 1);
-                    _reverseEIDs.AddAt(lastIndex) = value.Key;
+                    _reverseEIDs.AddAt(lastIndex) = value.key;
                 }
 
             _indexOfEntityInDenseList.Clear();

Core/Filters/LegacyFilterGroup.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 07e497071a7f310497abce732bb63447
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/FilteredIndices.cs → Core/Filters/LegacyFilteredIndices.cs

@@ -3,16 +3,19 @@ using Svelto.ECS.DataStructures;
 
 namespace Svelto.ECS
 {
-    public readonly struct FilteredIndices
+    public readonly struct LegacyFilteredIndices
     {
-        public FilteredIndices(NativeDynamicArrayCast<uint> denseListOfIndicesToEntityComponentArray)
+        public LegacyFilteredIndices(NativeDynamicArrayCast<uint> denseListOfIndicesToEntityComponentArray)
         {
             _denseListOfIndicesToEntityComponentArray = denseListOfIndicesToEntityComponentArray;
             _count                                    = _denseListOfIndicesToEntityComponentArray.count;
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public int Count() => _count;
+        public int count
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get { return _count; }
+        }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public uint Get(uint index) => _denseListOfIndicesToEntityComponentArray[index];

Core/Filters/LegacyFilteredIndices.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: abe8e2548ed3369c86a3a55c5d49429d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/LegacyGroupFilters.cs

@@ -0,0 +1,104 @@
+using Svelto.DataStructures;
+using Svelto.DataStructures.Native;
+
+namespace Svelto.ECS
+{
+    public struct LegacyGroupFilters
+    {
+        internal LegacyGroupFilters(SharedSveltoDictionaryNative<int, LegacyFilterGroup> legacyFilters, ExclusiveGroupStruct group)
+        {
+            this._legacyFilters = legacyFilters;
+            _group       = @group;
+        }
+
+        public ref LegacyFilterGroup GetFilter(int filterIndex)
+        {
+#if DEBUG && !PROFILE_SVELTO
+            if (_legacyFilters.isValid == false)
+                throw new ECSException($"trying to fetch not existing filters {filterIndex} group {_group.ToName()}");
+            if (_legacyFilters.ContainsKey(filterIndex) == false)
+                throw new ECSException($"trying to fetch not existing filters {filterIndex} group {_group.ToName()}");
+#endif
+            return ref _legacyFilters.GetValueByRef(filterIndex);
+        }
+
+        public bool HasFilter(int filterIndex) { return _legacyFilters.ContainsKey(filterIndex); }
+
+        public void ClearFilter(int filterIndex)
+        {
+            if (_legacyFilters.TryFindIndex(filterIndex, out var index))
+                _legacyFilters.GetValues(out _)[index].Clear();
+        }
+
+        public void ClearFilters()
+        {
+            foreach (var filter in _legacyFilters)
+                filter.value.Clear();
+        }
+
+        public bool TryGetFilter(int filterIndex, out LegacyFilterGroup legacyFilter)
+        {
+            return _legacyFilters.TryGetValue(filterIndex, out legacyFilter);
+        }
+
+        public SveltoDictionaryKeyValueEnumerator<int, LegacyFilterGroup, NativeStrategy<SveltoDictionaryNode<int>>, NativeStrategy<LegacyFilterGroup>
+          , NativeStrategy<int>> GetEnumerator()
+        {
+            return _legacyFilters.GetEnumerator();
+        }
+        
+        //Note the following methods are internal because I was pondering the idea to be able to return
+        //the list of LegacyGroupFilters linked to a specific filter ID. However this would mean to be able to
+        //maintain a revers map which at this moment seems too much and also would need the following
+        //method to be for ever internal (at this point in time I am not sure it's a good idea)
+        internal void DisposeFilter(int filterIndex)
+        {
+            if (_legacyFilters.TryFindIndex(filterIndex, out var index))
+            {
+                ref var filterGroup = ref _legacyFilters.GetValues(out _)[index];
+                
+                filterGroup.Dispose();
+
+                _legacyFilters.Remove(filterIndex);
+            }
+        }
+
+        internal void DisposeFilters()
+        {
+            //must release the native buffers!
+            foreach (var filter in _legacyFilters)
+                filter.value.Dispose();
+
+            _legacyFilters.FastClear();
+        }
+
+        internal ref LegacyFilterGroup CreateOrGetFilter(int filterID)
+        {
+            if (_legacyFilters.TryFindIndex(filterID, out var index) == false)
+            {
+                var orGetFilterForGroup = new LegacyFilterGroup(_group, filterID);
+
+                _legacyFilters[filterID] = orGetFilterForGroup;
+
+                return ref _legacyFilters.GetValueByRef(filterID);
+            }
+
+            return ref _legacyFilters.GetValues(out _)[index];
+        }
+
+        internal void Dispose()
+        {
+            foreach (var filter in _legacyFilters)
+            {
+                filter.value.Dispose();
+            }
+
+            _legacyFilters.Dispose();
+        }
+
+        readonly ExclusiveGroupStruct _group;
+
+        //filterID, filter
+        SharedSveltoDictionaryNative<int, LegacyFilterGroup> _legacyFilters;
+    }
+}

Core/Filters/LegacyGroupFilters.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: a2af880c69c83d14b104072836603014
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/NativeEntityFilterCollection.cs

@@ -0,0 +1,148 @@
+using System.Runtime.CompilerServices;
+using Svelto.DataStructures.Native;
+using Svelto.ECS.Native;
+
+namespace Svelto.ECS
+{
+    public struct NativeEntityFilterCollection<T> where T : unmanaged, IEntityComponent
+    {
+        internal NativeEntityFilterCollection(NativeEGIDMultiMapper<T> mmap)
+        {
+            _mmap            = mmap;
+            _filtersPerGroup = new SharedSveltoDictionaryNative<ExclusiveGroupStruct, GroupFilters>();
+        }
+
+        public NativeEntityFilterIterator<T> iterator => new NativeEntityFilterIterator<T>(this);
+
+        public void AddEntity(EGID egid)
+        {
+            AddEntity(egid, _mmap.GetIndex(egid));
+        }
+
+        public void RemoveEntity(EGID egid)
+        {
+            _filtersPerGroup[egid.groupID].Remove(egid.entityID);
+        }
+
+        public void Clear()
+        {
+            var filterSets = _filtersPerGroup.GetValues(out var count);
+            for (var i = 0; i < count; i++)
+            {
+                filterSets[i].Clear();
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        void AddEntity(EGID egid, uint toIndex)
+        {
+            if (_filtersPerGroup.TryGetValue(egid.groupID, out var groupFilter) == false)
+            {
+                groupFilter                    = new GroupFilters(32, egid.groupID);
+                _filtersPerGroup[egid.groupID] = groupFilter;
+            }
+
+            groupFilter.Add(egid.entityID, toIndex);
+        }
+
+        internal int groupCount => _filtersPerGroup.count;
+
+        internal GroupFilters GetGroup(int indexGroup)
+        {
+            DBC.ECS.Check.Require(indexGroup < _filtersPerGroup.count);
+            return _filtersPerGroup.GetValues(out _)[indexGroup];
+        }
+
+        internal void Dispose()
+        {
+            var filterSets = _filtersPerGroup.GetValues(out var count);
+            for (var i = 0; i < count; i++)
+            {
+                filterSets[i].Dispose();
+            }
+        }
+
+        readonly NativeEGIDMultiMapper<T> _mmap;
+        //double check if this needs to be shared
+        SharedSveltoDictionaryNative<ExclusiveGroupStruct, GroupFilters> _filtersPerGroup;
+
+        internal struct GroupFilters
+        {
+            internal GroupFilters(uint size, ExclusiveGroupStruct group)
+            {
+                _entityIDToDenseIndex = new SharedSveltoDictionaryNative<uint, uint>(size);
+                _indexToEntityId      = new SharedSveltoDictionaryNative<uint, uint>(size);
+                _group                = group;
+            }
+
+            internal void Add(uint entityId, uint entityIndex)
+            {
+                _entityIDToDenseIndex.Add(entityId, entityIndex);
+                _indexToEntityId.Add(entityIndex, entityId);
+            }
+
+            internal void Remove(uint entityId)
+            {
+                _indexToEntityId.Remove(_entityIDToDenseIndex[entityId]);
+                _entityIDToDenseIndex.Remove(entityId);
+            }
+
+            internal void RemoveWithSwapBack(uint entityId, uint entityIndex, uint lastIndex)
+            {
+                // Check if the last index is part of the filter as an entity, in that case
+                //we need to update the filter
+                if (entityIndex != lastIndex && _indexToEntityId.ContainsKey(lastIndex))
+                {
+                    uint lastEntityID = _indexToEntityId[lastIndex];
+
+                    _entityIDToDenseIndex[lastEntityID] = entityIndex;
+                    _indexToEntityId[entityIndex]       = lastEntityID;
+
+                    _indexToEntityId.Remove(lastIndex);
+                }
+                else
+                {
+                    // We don't need to check if the entityIndex is a part of the dictionary.
+                    // The Remove function will check for us.
+                    _indexToEntityId.Remove(entityIndex);
+                }
+
+                // We don't need to check if the entityID is part of the dictionary.
+                // The Remove function will check for us.
+                _entityIDToDenseIndex.Remove(entityId);
+            }
+
+            internal void Clear()
+            {
+                _indexToEntityId.FastClear();
+                _entityIDToDenseIndex.FastClear();
+            }
+
+            internal bool HasEntity(uint entityId) => _entityIDToDenseIndex.ContainsKey(entityId);
+
+            internal void Dispose()
+            {
+                _entityIDToDenseIndex.Dispose();
+                _indexToEntityId.Dispose();
+            }
+
+            internal EntityFilterIndices indices
+            {
+                get
+                {
+                    var values = _entityIDToDenseIndex.GetValues(out var count);
+                    return new EntityFilterIndices(values, count);
+                }
+            }
+
+            internal uint count => (uint)_entityIDToDenseIndex.count;
+
+            internal ExclusiveGroupStruct group => _group;
+
+            //double check if these need to be shared
+            SharedSveltoDictionaryNative<uint, uint> _indexToEntityId;
+            SharedSveltoDictionaryNative<uint, uint> _entityIDToDenseIndex;
+            readonly ExclusiveGroupStruct            _group;
+        }
+    }
+}

Core/Filters/NativeEntityFilterCollection.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 468b87cdba333e5d87a62d70715d374c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/Filters/NativeEntityFilterIterator.cs

@@ -0,0 +1,63 @@
+namespace Svelto.ECS
+{
+    public readonly ref struct NativeEntityFilterIterator<T> where T : unmanaged, IEntityComponent
+    {
+        internal NativeEntityFilterIterator(NativeEntityFilterCollection<T> filter)
+        {
+            _filter = filter;
+        }
+
+        public Iterator GetEnumerator() => new Iterator(_filter);
+
+        readonly NativeEntityFilterCollection<T> _filter;
+
+        public ref struct Iterator
+        {
+            internal Iterator(NativeEntityFilterCollection<T>  filter)
+            {
+                _filter     = filter;
+                _indexGroup = -1;
+                _current    = default;
+            }
+
+            public bool MoveNext()
+            {
+                while (++_indexGroup < _filter.groupCount)
+                {
+                    _current = _filter.GetGroup(_indexGroup);
+
+                    if (_current.count > 0) break;
+                }
+
+                return _indexGroup < _filter.groupCount;
+            }
+
+            public void Reset()
+            {
+                _indexGroup = -1;
+            }
+
+            public RefCurrent Current => new RefCurrent(_current);
+
+            int                                          _indexGroup;
+            readonly NativeEntityFilterCollection<T>     _filter;
+            NativeEntityFilterCollection<T>.GroupFilters _current;
+        }
+
+        public readonly ref struct RefCurrent
+        {
+            internal RefCurrent(NativeEntityFilterCollection<T>.GroupFilters filter)
+            {
+                _filter = filter;
+            }
+
+            public void Deconstruct(out EntityFilterIndices indices, out ExclusiveGroupStruct group)
+            {
+                indices = _filter.indices;
+                group   = _filter.group;
+            }
+
+            readonly NativeEntityFilterCollection<T>.GroupFilters _filter;
+        }
+    }
+}

Core/Filters/NativeEntityFilterIterator.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: a699fb7580693a9d96efd75dd7f8858f
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Core/GlobalTypeID.cs

@@ -1,6 +1,5 @@
 using System.Threading;
 using Svelto.Common;
-using Svelto.DataStructures;
 using Svelto.ECS.DataStructures;
 
 namespace Svelto.ECS
@@ -23,7 +22,7 @@ namespace Svelto.ECS
     {
         static Filler()
         {
-            DBC.ECS.Check.Require(TypeCache<T>.isUnmanaged == true, "invalid type used");
+            DBC.ECS.Check.Require(TypeType.isUnmanaged<T>() == true, "invalid type used");
         }
 
         //it's an internal interface
@@ -35,28 +34,20 @@ namespace Svelto.ECS
         }
     }
 
+#if UNITY_NATIVE //at the moment I am still considering NativeOperations useful only for Unity
     static class EntityComponentID<T>
     {
-#if UNITY_NATIVE
         internal static readonly Unity.Burst.SharedStatic<uint> ID =
             Unity.Burst.SharedStatic<uint>.GetOrCreate<GlobalTypeID, T>();
-#else
-        internal struct SharedStatic
-        {
-            public uint Data;
-        }
-
-        internal static SharedStatic ID;
-#endif
     }
 
     static class EntityComponentIDMap
     {
-        static readonly FasterList<IFiller> TYPE_IDS;
+        static readonly Svelto.DataStructures.FasterList<IFiller> TYPE_IDS;
 
         static EntityComponentIDMap()
         {
-            TYPE_IDS = new FasterList<IFiller>();
+            TYPE_IDS = new Svelto.DataStructures.FasterList<IFiller>();
         }
 
         internal static void Register<T>(IFiller entityBuilder) where T : struct, IEntityComponent
@@ -67,4 +58,5 @@ namespace Svelto.ECS
 
         internal static IFiller GetTypeFromID(uint typeId) { return TYPE_IDS[typeId]; }
     }
+#endif
 }

Core/GroupHashMap.cs

@@ -6,12 +6,12 @@ using Svelto.ECS.Serialization;
 
 namespace Svelto.ECS
 {
-    static class GroupHashMap
+    public static class GroupHashMap
     {
         /// <summary>
         /// c# Static constructors are guaranteed to be thread safe
         /// </summary>
-        public static void Init()
+        internal static void Init()
         {
             List<Assembly> assemblies = AssemblyUtility.GetCompatibleAssemblies();
             foreach (Assembly assembly in assemblies)
@@ -20,41 +20,61 @@ namespace Svelto.ECS
                 {
                     var typeOfExclusiveGroup       = typeof(ExclusiveGroup);
                     var typeOfExclusiveGroupStruct = typeof(ExclusiveGroupStruct);
+                    var typeOfExclusiveBuildGroup  = typeof(ExclusiveBuildGroup);
 
                     foreach (Type type in AssemblyUtility.GetTypesSafe(assembly))
                     {
-                        if (type != null && type.IsClass && type.IsSealed
-                         && type.IsAbstract) //this means only static classes
+                        CheckForGroupCompounds(type);
+
+                        if (type != null && type.IsClass && type.IsSealed &&
+                            type.IsAbstract) //IsClass and IsSealed and IsAbstract means only static classes
                         {
+                            var subClasses = type.GetNestedTypes();
+
+                            foreach (var subclass in subClasses)
+                            {
+                                CheckForGroupCompounds(subclass);
+                            }
+
                             var fields = type.GetFields();
 
                             foreach (var field in fields)
                             {
-                                if (field.IsStatic)
+                                if (field.IsStatic 
+                                     && (typeOfExclusiveGroup.IsAssignableFrom(field.FieldType) 
+                                     || typeOfExclusiveGroupStruct.IsAssignableFrom(field.FieldType) 
+                                     || typeOfExclusiveBuildGroup.IsAssignableFrom(field.FieldType)))
                                 {
+                                    uint groupID;
+
                                     if (typeOfExclusiveGroup.IsAssignableFrom(field.FieldType))
                                     {
-                                        var group = (ExclusiveGroup) field.GetValue(null);
-#if DEBUG
-                                        GroupNamesMap.idToName[@group] =
-                                            $"{$"{type.FullName}.{field.Name}"} {group.id})";
-#endif
-                                        //The hashname is independent from the actual group ID. this is fundamental because it is want
-                                        //guarantees the hash to be the same across different machines
-                                        RegisterGroup(group, $"{type.FullName}.{field.Name}");
+                                        var group = (ExclusiveGroup)field.GetValue(null);
+                                        groupID = ((ExclusiveGroupStruct)@group).id;
+                                    }
+                                    else
+                                    if (typeOfExclusiveGroupStruct.IsAssignableFrom(field.FieldType))
+                                    {
+                                        var group = (ExclusiveGroupStruct)field.GetValue(null);
+                                        groupID = @group.id;
                                     }
                                     else
-                                        if (typeOfExclusiveGroupStruct.IsAssignableFrom(field.FieldType))
-                                        {
-                                            var group = (ExclusiveGroupStruct) field.GetValue(null);
-#if DEBUG
+                                    {
+                                        var group = (ExclusiveBuildGroup)field.GetValue(null);
+                                        groupID = ((ExclusiveGroupStruct)@group).id;
+                                    }
+
+                                    {
+                                        ExclusiveGroupStruct group = new ExclusiveGroupStruct(groupID);
+#if DEBUG && !PROFILE_SVELTO
+                                        if (GroupNamesMap.idToName.ContainsKey(@group) == false)
                                             GroupNamesMap.idToName[@group] =
-                                                $"{$"{type.FullName}.{field.Name}"} {group.id})";
+                                                $"{type.FullName}.{field.Name} {@group.id})";
 #endif
-                                            //The hashname is independent from the actual group ID. this is fundamental because it is want
-                                            //guarantees the hash to be the same across different machines
-                                            RegisterGroup(@group, $"{type.FullName}.{field.Name}");
-                                        }
+                                        //The hashname is independent from the actual group ID. this is fundamental because it is want
+                                        //guarantees the hash to be the same across different machines
+                                        RegisterGroup(@group, $"{type.FullName}.{field.Name}");
+                                    }
                                 }
                             }
                         }
@@ -69,6 +89,16 @@ namespace Svelto.ECS
             }
         }
 
+        static void CheckForGroupCompounds(Type type)
+        {
+            if (typeof(ITouchedByReflection).IsAssignableFrom(type))
+            {
+                //this wil call the static constructor, but only once. Static constructors won't be called
+                //more than once with this
+                System.Runtime.CompilerServices.RuntimeHelpers.RunClassConstructor(type.BaseType.TypeHandle);
+            }
+        }
+
         /// <summary>
         /// The hashname is independent from the actual group ID. this is fundamental because it is want
         /// guarantees the hash to be the same across different machines
@@ -78,7 +108,8 @@ namespace Svelto.ECS
         /// <exception cref="ECSException"></exception>
         public static void RegisterGroup(ExclusiveGroupStruct exclusiveGroupStruct, string name)
         {
-            //Group already registered by another field referencing the same group
+            //Group already registered by another field referencing the same group, can happen because
+            //the group poked is a group compound which static constructor is already been called at this point
             if (_hashByGroups.ContainsKey(exclusiveGroupStruct))
                 return;
 
@@ -93,9 +124,9 @@ namespace Svelto.ECS
             _hashByGroups.Add(exclusiveGroupStruct, nameHash);
         }
 
-        public static uint GetHashFromGroup(ExclusiveGroupStruct groupStruct)
+        internal static uint GetHashFromGroup(ExclusiveGroupStruct groupStruct)
         {
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
             if (_hashByGroups.ContainsKey(groupStruct) == false)
                 throw new ECSException($"Attempted to get hash from unregistered group {groupStruct}");
 #endif
@@ -103,9 +134,9 @@ namespace Svelto.ECS
             return _hashByGroups[groupStruct];
         }
 
-        public static ExclusiveGroupStruct GetGroupFromHash(uint groupHash)
+        internal static ExclusiveGroupStruct GetGroupFromHash(uint groupHash)
         {
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
             if (_groupsByHash.ContainsKey(groupHash) == false)
                 throw new ECSException($"Attempted to get group from unregistered hash {groupHash}");
 #endif

Core/GroupNamesMap.cs

@@ -3,12 +3,12 @@ using Svelto.ECS;
 
 static class GroupNamesMap
 {
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
     static GroupNamesMap() { idToName = new Dictionary<ExclusiveGroupStruct, string>(); }
 
     internal static readonly Dictionary<ExclusiveGroupStruct, string> idToName;
 #endif
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
     public static string ToName(this in ExclusiveGroupStruct group)
     {
         Dictionary<ExclusiveGroupStruct, string> idToName = GroupNamesMap.idToName;

Core/Groups/ExclusiveBuildGroup.cs

@@ -16,7 +16,7 @@ namespace Svelto.ECS
         {
             return new ExclusiveBuildGroup(group);
         }
-        
+
         public static implicit operator ExclusiveGroupStruct(ExclusiveBuildGroup group)
         {
             return group.group;

Core/Groups/ExclusiveGroup.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Runtime.CompilerServices;
 
 #pragma warning disable 660,661
 
@@ -43,7 +42,7 @@ namespace Svelto.ECS
         public ExclusiveGroup(ushort range)
         {
             _group = ExclusiveGroupStruct.GenerateWithRange(range);
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
             _range = range;
 #endif
         }
@@ -55,7 +54,7 @@ namespace Svelto.ECS
 
         public static ExclusiveGroupStruct operator+(ExclusiveGroup @group, uint b)
         {
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
             if (@group._range == 0)
                 throw new ECSException($"Adding values to a not ranged ExclusiveGroup: {@group.id}");
             if (b >= @group._range)
@@ -83,7 +82,7 @@ namespace Svelto.ECS
         static readonly Dictionary<string, ExclusiveGroupStruct> _knownGroups =
             new Dictionary<string, ExclusiveGroupStruct>();
 
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
         readonly ushort _range;
 #endif
         readonly ExclusiveGroupStruct _group;

Core/Groups/ExclusiveGroupStruct.cs

@@ -1,10 +1,12 @@
 using System;
+using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Threading;
 
 namespace Svelto.ECS
 {
+    [DebuggerDisplay("{ToString()}")]
     [StructLayout(LayoutKind.Explicit, Size = 4)]
     //the type doesn't implement IEqualityComparer, what implements it is a custom comparer
     public readonly struct ExclusiveGroupStruct : IEquatable<ExclusiveGroupStruct>, IComparable<ExclusiveGroupStruct>

Core/Groups/GroupCompound.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using System.Threading;
 using Svelto.DataStructures;
@@ -18,25 +17,31 @@ namespace Svelto.ECS
         internal static readonly ThreadLocal<bool> skipStaticCompoundConstructorsWith2Tags = new ThreadLocal<bool>();
     }
 
-    public abstract class GroupCompound<G1, G2, G3, G4> where G1 : GroupTag<G1>
-                                                        where G2 : GroupTag<G2>
-                                                        where G3 : GroupTag<G3>
-                                                        where G4 : GroupTag<G4>
+    interface ITouchedByReflection
+    {
+    }
+
+    public abstract class GroupCompound<G1, G2, G3, G4> : ITouchedByReflection where G1 : GroupTag<G1>
+        where G2 : GroupTag<G2>
+        where G3 : GroupTag<G3>
+        where G4 : GroupTag<G4>
     {
         static GroupCompound()
         {
             //avoid race conditions if compounds are using on multiple thread
-            if (Interlocked.CompareExchange(ref isInitializing, 1, 0) == 0
-             && GroupCompoundInitializer.skipStaticCompoundConstructorsWith4Tags.Value == false)
+            if (Interlocked.CompareExchange(ref isInitializing, 1, 0) == 0 &&
+                GroupCompoundInitializer.skipStaticCompoundConstructorsWith4Tags.Value == false)
             {
-                var group = new ExclusiveGroup(); //todo: it's a bit of a waste to create a class here even if this is a static constructor
+                var
+                    group =
+                        new ExclusiveGroup(); //todo: it's a bit of a waste to create a class here even if this is a static constructor
 
                 _Groups = new FasterList<ExclusiveGroupStruct>(1);
                 _Groups.Add(group);
-                
-#if DEBUG
+
+#if DEBUG && !PROFILE_SVELTO
                 var name =
-                    $"Compound: {typeof(G1).Name}-{typeof(G2).Name}-{typeof(G3).Name}-{typeof(G4).Name} ID {(uint) group.id}";
+                    $"Compound: {typeof(G1).Name}-{typeof(G2).Name}-{typeof(G3).Name}-{typeof(G4).Name} ID {(uint)group.id}";
                 GroupNamesMap.idToName[group] = name;
 #endif
                 //The hashname is independent from the actual group ID. this is fundamental because it is want
@@ -133,16 +138,16 @@ namespace Svelto.ECS
 
         internal static void Add(ExclusiveGroupStruct group)
         {
-#if DEBUG && !PROFILE_SVELTO            
+#if DEBUG && !PROFILE_SVELTO
             for (var i = 0; i < _Groups.count; ++i)
                 if (_Groups[i] == group)
-                    throw new Exception("this test must be transformed in unit test");
+                    throw new System.Exception("this test must be transformed in unit test");
 #endif
 
             _Groups.Add(group);
             _GroupsHashSet.Add(group);
         }
-        
+
         static readonly FasterList<ExclusiveGroupStruct> _Groups;
         static readonly HashSet<ExclusiveGroupStruct>    _GroupsHashSet;
 
@@ -150,21 +155,24 @@ namespace Svelto.ECS
         static int isInitializing;
     }
 
-    public abstract class GroupCompound<G1, G2, G3>
-        where G1 : GroupTag<G1> where G2 : GroupTag<G2> where G3 : GroupTag<G3>
+    public abstract class GroupCompound<G1, G2, G3> : ITouchedByReflection where G1 : GroupTag<G1>
+        where G2 : GroupTag<G2>
+        where G3 : GroupTag<G3>
     {
         static GroupCompound()
         {
-            if (Interlocked.CompareExchange(ref isInitializing, 1, 0) == 0
-             && GroupCompoundInitializer.skipStaticCompoundConstructorsWith3Tags.Value == false)
+            if (Interlocked.CompareExchange(ref isInitializing, 1, 0) == 0 &&
+                GroupCompoundInitializer.skipStaticCompoundConstructorsWith3Tags.Value == false)
             {
-                var group = new ExclusiveGroup(); //todo: it's a bit of a waste to create a class here even if this is a static constructor
+                var
+                    group =
+                        new ExclusiveGroup(); //todo: it's a bit of a waste to create a class here even if this is a static constructor
 
                 _Groups = new FasterList<ExclusiveGroupStruct>(1);
                 _Groups.Add(group);
 
-#if DEBUG
-                var name = $"Compound: {typeof(G1).Name}-{typeof(G2).Name}-{typeof(G3).Name} ID {(uint) group.id}";
+#if DEBUG && !PROFILE_SVELTO
+                var name = $"Compound: {typeof(G1).Name}-{typeof(G2).Name}-{typeof(G3).Name} ID {(uint)group.id}";
                 GroupNamesMap.idToName[group] = name;
 #endif
                 //The hashname is independent from the actual group ID. this is fundamental because it is want
@@ -215,16 +223,16 @@ namespace Svelto.ECS
 
         internal static void Add(ExclusiveGroupStruct group)
         {
-#if DEBUG && !PROFILE_SVELTO            
+#if DEBUG && !PROFILE_SVELTO
             for (var i = 0; i < _Groups.count; ++i)
                 if (_Groups[i] == group)
-                    throw new Exception("this test must be transformed in unit test");
+                    throw new System.Exception("this test must be transformed in unit test");
 #endif
 
             _Groups.Add(group);
             _GroupsHashSet.Add(group);
         }
-        
+
         static readonly FasterList<ExclusiveGroupStruct> _Groups;
         static readonly HashSet<ExclusiveGroupStruct>    _GroupsHashSet;
 
@@ -232,19 +240,21 @@ namespace Svelto.ECS
         static int isInitializing;
     }
 
-    public abstract class GroupCompound<G1, G2> where G1 : GroupTag<G1> where G2 : GroupTag<G2>
+    public abstract class GroupCompound<G1, G2> : ITouchedByReflection where G1 : GroupTag<G1> where G2 : GroupTag<G2>
     {
         static GroupCompound()
         {
-            if (Interlocked.CompareExchange(ref isInitializing, 1, 0) == 0
-             && GroupCompoundInitializer.skipStaticCompoundConstructorsWith2Tags.Value == false)
+            if (Interlocked.CompareExchange(ref isInitializing, 1, 0) == 0 &&
+                GroupCompoundInitializer.skipStaticCompoundConstructorsWith2Tags.Value == false)
             {
-                var group = new ExclusiveGroup(); //todo: it's a bit of a waste to create a class here even if this is a static constructor
+                var
+                    group =
+                        new ExclusiveGroup(); //todo: it's a bit of a waste to create a class here even if this is a static constructor
 
                 _Groups = new FasterList<ExclusiveGroupStruct>(1);
                 _Groups.Add(group);
 
-#if DEBUG
+#if DEBUG && !PROFILE_SVELTO
                 GroupNamesMap.idToName[group] = $"Compound: {typeof(G1).Name}-{typeof(G2).Name} ID {group.id}";
 #endif
                 //The hashname is independent from the actual group ID. this is fundamental because it is want
@@ -276,16 +286,16 @@ namespace Svelto.ECS
 
         internal static void Add(ExclusiveGroupStruct group)
         {
-#if DEBUG && !PROFILE_SVELTO            
+#if DEBUG && !PROFILE_SVELTO
             for (var i = 0; i < _Groups.count; ++i)
                 if (_Groups[i] == group)
-                    throw new Exception("this test must be transformed in unit test");
-#endif            
+                    throw new System.Exception("this test must be transformed in unit test");
+#endif
 
             _Groups.Add(group);
             _GroupsHashSet.Add(group);
         }
-        
+
         static readonly FasterList<ExclusiveGroupStruct> _Groups;
         static readonly HashSet<ExclusiveGroupStruct>    _GroupsHashSet;
 
@@ -299,7 +309,7 @@ namespace Svelto.ECS
     ///     groups with the same adjective, a group tag needs to hold all the groups sharing it.
     /// </summary>
     /// <typeparam name="T"></typeparam>
-    public abstract class GroupTag<T> where T : GroupTag<T>
+    public abstract class GroupTag<T> : ITouchedByReflection where T : GroupTag<T>
     {
         static GroupTag()
         {
@@ -308,12 +318,13 @@ namespace Svelto.ECS
                 var group = new ExclusiveGroup();
                 _Groups.Add(group);
 
-#if DEBUG
-                var typeInfo         = typeof(T);
-                var name             = $"Compound: {typeInfo.Name} ID {(uint) group.id}";
-                
+#if DEBUG && !PROFILE_SVELTO
+                var typeInfo = typeof(T);
+                var name     = $"Compound: {typeInfo.Name} ID {(uint)group.id}";
+
                 var typeInfoBaseType = typeInfo.BaseType;
-                if (typeInfoBaseType.GenericTypeArguments[0] != typeInfo) //todo: this should shield from using a pattern different than public class GROUP_NAME : GroupTag<GROUP_NAME> {} however I am not sure it's working
+                if (typeInfoBaseType.GenericTypeArguments[0] !=
+                    typeInfo) //todo: this should shield from using a pattern different than public class GROUP_NAME : GroupTag<GROUP_NAME> {} however I am not sure it's working
                     throw new ECSException("Invalid Group Tag declared");
 
                 GroupNamesMap.idToName[group] = name;
@@ -339,16 +350,16 @@ namespace Svelto.ECS
         //Each time a new combination of group tags is found a new group is added.
         internal static void Add(ExclusiveGroupStruct group)
         {
-#if DEBUG && !PROFILE_SVELTO            
+#if DEBUG && !PROFILE_SVELTO
             for (var i = 0; i < _Groups.count; ++i)
                 if (_Groups[i] == group)
-                    throw new Exception("this test must be transformed in unit test");
+                    throw new System.Exception("this test must be transformed in unit test");
 #endif
 
             _Groups.Add(group);
             _GroupsHashSet.Add(group);
         }
-        
+
         static readonly FasterList<ExclusiveGroupStruct> _Groups = new FasterList<ExclusiveGroupStruct>(1);
         static readonly HashSet<ExclusiveGroupStruct>    _GroupsHashSet;
 

Core/Groups/NamedExclusiveGroup.cs

@@ -13,7 +13,7 @@ namespace Svelto.ECS
 
         static NamedExclusiveGroup()
         {
-#if DEBUG        
+#if DEBUG && !PROFILE_SVELTO
             GroupNamesMap.idToName[Group] = $"{name} ID {Group.id}";
 #endif            
             //The hashname is independent from the actual group ID. this is fundamental because it is want

Core/Hybrid/IEntityViewComponent.cs

@@ -3,7 +3,10 @@ namespace Svelto.ECS.Hybrid
     public interface IManagedComponent:IEntityComponent
     {}
     
-    public interface IEntityViewComponent:IManagedComponent, INeedEGID
+    public interface IEntityViewComponent:IManagedComponent
+#if SLOW_SVELTO_SUBMISSION
+        ,INeedEGID
+#endif    
     {}
 }
 

Core/Hybrid/ValueReference.cs

@@ -1,37 +1,29 @@
-using System;
 using System.Runtime.InteropServices;
 
 namespace Svelto.ECS.Hybrid
 {
     /// <summary>
-    /// an ECS dirty trick to hold the reference to an object. this is component can be used in an engine
-    /// managing an OOP abstraction layer. It's need is quite rare though! An example is found at
-    /// https://github.com/sebas77/Svelto.MiniExamples/blob/master/Example6-Unity%20Hybrid-OOP%20Abstraction/Assets/Code/WithEntityViewComponent/Descriptors/TransformImplementor.cs
-    /// All other uses must be considered an abuse
-    /// as the object can be casted back to it's real type only by an OOP Abstraction Layer Engine:
-    /// https://www.sebaslab.com/oop-abstraction-layer-in-a-ecs-centric-application/ 
+    /// ValueReference is the only way to store a reference inside an Implementor. To stop any abuse
+    /// the reference must be an implementor and converted back to an implementor.
+    /// The OOP abstraction layer that knows about the implementor than can cast it to the real type
     /// </summary>
     /// <typeparam name="T"></typeparam>
-    public struct ValueReference<T> : IDisposable where T:class, IImplementor
+    public struct ValueReference<T> : IValueReferenceInternal where T:class
     {
-        static ValueReference()
-        {
-            DBC.ECS.Check.Require(typeof(T).IsInterface == true, "ValueReference type can be only pure interface implementing IImplementor");
-        }
-
         public ValueReference(T obj) { _pointer = GCHandle.Alloc(obj, GCHandleType.Normal); }
 
-        public W Convert<W>(W implementer) where W:T
+        public T ConvertAndDispose<W>(W implementer) where W:IImplementor 
         {
             var pointerTarget = _pointer.Target;
-            return (W)pointerTarget;
-        }
-
-        public void Dispose()
-        {
             _pointer.Free();
+            return (T)pointerTarget;
         }
+
+        public bool isDefault => _pointer.IsAllocated == false;
         
         GCHandle    _pointer;
     }
+
+    // Used to validate the use of this struct on the component builder check fields.
+    internal interface IValueReferenceInternal {}
 }

Core/IEngine.cs

@@ -2,41 +2,128 @@ using Svelto.ECS.Internal;
 
 namespace Svelto.ECS.Internal
 {
-    public interface IReactEngine: IEngine
-    {}
+    public interface IReactEngine : IEngine
+    {
+    }
+#region legacy interfaces
+    /// <summary>
+    /// This is now considered legacy and it will be deprecated in future
+    /// </summary>
+    public interface IReactOnAdd : IReactEngine
+    {
+    }
     
-    public interface IReactOnAddAndRemove : IReactEngine
-    {}
+    /// <summary>
+    /// This is now considered legacy and it will be deprecated in future
+    /// </summary>
+    public interface IReactOnRemove : IReactEngine
+    {
+    }
     
-    public interface IReactOnDispose : IReactEngine
-    {}
-
+    /// <summary>
+    /// This is now considered legacy and it will be deprecated in future
+    /// </summary>
     public interface IReactOnSwap : IReactEngine
-    {}
+    {
+    }
+#endregion    
+
+    public interface IReactOnAddEx : IReactEngine
+    {
+    }
+
+    public interface IReactOnRemoveEx : IReactEngine
+    {
+    }
+
+    public interface IReactOnSwapEx : IReactEngine
+    {
+    }
+
+    public interface IReactOnDispose : IReactEngine
+    {
+    }
 }
 
 namespace Svelto.ECS
 {
     public interface IEngine
-    {}
-    
-    public interface IReactOnAddAndRemove<T> : IReactOnAddAndRemove where T : IEntityComponent
+    {
+    }
+
+    public interface IGetReadyEngine : IEngine
+    {
+        void Ready();
+    }
+
+    public interface IQueryingEntitiesEngine : IGetReadyEngine
+    {
+        EntitiesDB entitiesDB { set; }
+    }
+
+    /// <summary>
+    /// Interface to mark an Engine as reacting on entities added
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public interface IReactOnAdd<T> : IReactOnAdd where T : IEntityComponent
     {
         void Add(ref T entityComponent, EGID egid);
+    }
+
+    public interface IReactOnAddEx<T> : IReactOnAddEx where T : struct, IEntityComponent
+    {
+        void Add((uint start, uint end) rangeOfEntities, in EntityCollection<T> collection,
+            ExclusiveGroupStruct groupID);
+    }
+
+    /// <summary>
+    /// Interface to mark an Engine as reacting on entities removed
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public interface IReactOnRemove<T> : IReactOnRemove where T : IEntityComponent
+    {
         void Remove(ref T entityComponent, EGID egid);
     }
-    
+
+    public interface IReactOnRemoveEx<T> : IReactOnRemoveEx where T : struct, IEntityComponent
+    {
+        void Remove((uint start, uint end) rangeOfEntities, in EntityCollection<T> collection,
+            ExclusiveGroupStruct groupID);
+    }
+
+    public interface IReactOnAddAndRemove<T> : IReactOnAdd<T>, IReactOnRemove<T> where T : IEntityComponent
+    {
+    }
+
+    /// <summary>
+    /// Interface to mark an Engine as reacting on engines root disposed.
+    /// It can work together with IReactOnRemove which normally is not called on enginesroot disposed
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
     public interface IReactOnDispose<T> : IReactOnDispose where T : IEntityComponent
     {
         void Remove(ref T entityComponent, EGID egid);
     }
-    
+
+    /// <summary>
+    /// Interface to mark an Engine as reacting to entities swapping group
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
     public interface IReactOnSwap<T> : IReactOnSwap where T : IEntityComponent
     {
         void MovedTo(ref T entityComponent, ExclusiveGroupStruct previousGroup, EGID egid);
     }
 
-    public interface IReactOnSubmission:IReactEngine
+    public interface IReactOnSwapEx<T> : IReactOnSwapEx where T : struct, IEntityComponent
+    {
+        void MovedTo((uint start, uint end) rangeOfEntities, in EntityCollection<T> collection,
+            ExclusiveGroupStruct fromGroup, ExclusiveGroupStruct toGroup);
+    }
+
+    /// <summary>
+    /// Interface to mark an Engine as reacting after each entities submission phase
+    /// </summary>
+    public interface IReactOnSubmission : IReactEngine
     {
         void EntitiesSubmitted();
     }

Core/IEntityFactory.cs

@@ -39,25 +39,27 @@ namespace Svelto.ECS
         /// <param name="ed"></param>
         /// <param name="implementors"></param>
         EntityInitializer BuildEntity<T>(uint entityID, ExclusiveBuildGroup groupStructId,
-                                               IEnumerable<object> implementors = null)
-            where T : IEntityDescriptor, new();
+            IEnumerable<object> implementors = null,
+            [System.Runtime.CompilerServices.CallerMemberName] string caller = null) where T : IEntityDescriptor, new();
 
-        EntityInitializer BuildEntity<T>(EGID egid, IEnumerable<object> implementors = null)
-            where T : IEntityDescriptor, new();
+        EntityInitializer BuildEntity<T>(EGID egid, IEnumerable<object> implementors = null,
+            [System.Runtime.CompilerServices.CallerMemberName] string caller = null) where T : IEntityDescriptor, new();
 
-        EntityInitializer BuildEntity<T>(uint entityID, ExclusiveBuildGroup groupStructId,
-                                               T descriptorEntity, IEnumerable<object>  implementors = null)
-            where T : IEntityDescriptor;
+        EntityInitializer BuildEntity<T>(uint entityID, ExclusiveBuildGroup groupStructId, T descriptorEntity,
+            IEnumerable<object> implementors = null,
+            [System.Runtime.CompilerServices.CallerMemberName] string caller = null) where T : IEntityDescriptor;
 
-        EntityInitializer BuildEntity<T>(EGID egid, T entityDescriptor, IEnumerable<object> implementors = null)
-            where T : IEntityDescriptor;
+        EntityInitializer BuildEntity<T>(EGID egid, T entityDescriptor, IEnumerable<object> implementors = null,
+            [System.Runtime.CompilerServices.CallerMemberName] string caller = null) where T : IEntityDescriptor;
 
         //Todo: analyze if this can be internal or just related to serialization
-        EntityInitializer BuildEntity
-            (EGID egid, IComponentBuilder[] componentsToBuild, Type descriptorType, IEnumerable<object> implementors = null);
+        EntityInitializer BuildEntity(EGID egid, IComponentBuilder[] componentsToBuild, Type descriptorType,
+            IEnumerable<object> implementors = null,
+            [System.Runtime.CompilerServices.CallerMemberName] string caller = null);
 
 #if UNITY_NATIVE
-        Svelto.ECS.Native.NativeEntityFactory ToNative<T>(string callerName) where T : IEntityDescriptor, new();
-#endif        
+        Svelto.ECS.Native.NativeEntityFactory ToNative<T>([System.Runtime.CompilerServices.CallerMemberName] string callerName
+ = null) where T : IEntityDescriptor, new();
+#endif
     }
 }

Core/IEntityFunctions.cs

@@ -1,3 +1,5 @@
+using System.Runtime.CompilerServices;
+
 namespace Svelto.ECS
 {
     public interface IEntityFunctions
@@ -5,28 +7,19 @@ namespace Svelto.ECS
         //being entity ID globally not unique, the group must be specified when
         //an entity is removed. Not specifying the group will attempt to remove
         //the entity from the special standard group.
-        void RemoveEntity<T>(uint entityID, ExclusiveBuildGroup groupID) where T : IEntityDescriptor, new();
-        void RemoveEntity<T>(EGID entityegid) where T : IEntityDescriptor, new();
+        void RemoveEntity<T>(uint entityID, ExclusiveBuildGroup groupID , [CallerMemberName] string caller = null) where T : IEntityDescriptor, new();
+        void RemoveEntity<T>(EGID entityegid , [CallerMemberName] string caller = null) where T : IEntityDescriptor, new();
         
-        void RemoveEntitiesFromGroup(ExclusiveBuildGroup groupID);
-
-        void SwapEntitiesInGroup<T>(ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID)  where T : IEntityDescriptor, new();
-
-        void SwapEntityGroup<T>(uint entityID, ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID)
-            where T : IEntityDescriptor, new();
-
-        void SwapEntityGroup<T>(EGID fromID, ExclusiveBuildGroup toGroupID) where T : IEntityDescriptor, new();
-
-        void SwapEntityGroup<T>(EGID fromID, ExclusiveBuildGroup fromGroup, ExclusiveBuildGroup toGroupID)
-            where T : IEntityDescriptor, new();
-
-        void SwapEntityGroup<T>(EGID fromID, EGID toId) where T : IEntityDescriptor, new();
+        void RemoveEntitiesFromGroup(ExclusiveBuildGroup groupID , [CallerMemberName] string caller = null);
+        void SwapEntitiesInGroup(ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID, [CallerMemberName] string caller = null);
 
-        void SwapEntityGroup<T>(EGID fromID, EGID toId, ExclusiveBuildGroup mustBeFromGroup)
-            where T : IEntityDescriptor, new();
+        void SwapEntityGroup<T>(uint entityID, ExclusiveBuildGroup fromGroupID, ExclusiveBuildGroup toGroupID, [CallerMemberName] string caller = null) where T : IEntityDescriptor, new();
+        void SwapEntityGroup<T>(EGID fromEGID, ExclusiveBuildGroup toGroupID, [CallerMemberName] string caller = null) where T : IEntityDescriptor, new();
+        void SwapEntityGroup<T>(EGID fromEGID, EGID toEGID, [CallerMemberName] string caller = null)where T : IEntityDescriptor, new();
+        void SwapEntityGroup<T>(EGID fromEGID, EGID toEGID, ExclusiveBuildGroup mustBeFromGroup, [CallerMemberName] string caller = null) where T : IEntityDescriptor, new();
 #if UNITY_NATIVE
-        Svelto.ECS.Native.NativeEntityRemove                 ToNativeRemove<T>(string memberName)  where T : IEntityDescriptor, new();
-        Svelto.ECS.Native.NativeEntitySwap ToNativeSwap<T>(string memberName)  where T : IEntityDescriptor, new();
+        Svelto.ECS.Native.NativeEntityRemove ToNativeRemove<T>(string memberName)  where T : IEntityDescriptor, new();
+        Svelto.ECS.Native.NativeEntitySwap  ToNativeSwap<T>(string memberName)  where T : IEntityDescriptor, new();
 #endif        
     }
 }

Core/INeedEGID.cs

@@ -1,13 +1,16 @@
+#if SLOW_SVELTO_SUBMISSION
 namespace Svelto.ECS
 {
     /// <summary>
     /// use INeedEGID on an IEntityComponent only if you need the EGID. consider using EGIDComponent instead
     /// INeedEGID and EGIDComponent will become probably obsolete once QueryEntities will be able to return
     /// also the EGIDs to iterate upon
+    /// This is set to become obsolete at a given point
     /// </summary>
     public interface INeedEGID
     {
         //The set is used only by the framework, but it must stay there
         EGID ID { get; set; }
     }
-}
+}
+#endif

Core/INeedEntityReference.cs

@@ -1,5 +1,4 @@
-using Svelto.ECS.Reference;
-
+#if SLOW_SVELTO_SUBMISSION
 namespace Svelto.ECS
 {
     /// <summary>
@@ -7,9 +6,11 @@ namespace Svelto.ECS
     /// It currently exist because of the publisher/consumer behavior, but the publisher/consumer must not be
     /// considered an ECS pattern.
     /// Other uses are invalid.
+    /// It will become obsolete over the time
     /// </summary>
     public interface INeedEntityReference
     {
         EntityReference selfReference { get; set; }
     }
-}
+}
+#endif

Core/IQueryingEntitiesEngine.cs

@@ -1,9 +0,0 @@
-namespace Svelto.ECS
-{
-    public interface IQueryingEntitiesEngine : IEngine
-    {
-        EntitiesDB entitiesDB { set; }
-
-        void Ready();
-    }
-}

Core/IQueryingEntitiesEngine.cs.meta

@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 939e4e90b2f73bf4b7acc815acf6a808
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 

Core/QueryGroups.cs

@@ -5,72 +5,85 @@ using Svelto.DataStructures;
 
 namespace Svelto.ECS.Experimental
 {
-    struct GroupsList
+    internal struct GroupsList
     {
-        static GroupsList()
+        public static GroupsList Init()
         {
-            groups = new FasterList<ExclusiveGroupStruct>();
-            sets   = new HashSet<ExclusiveGroupStruct>();
-        }
+            var group = new GroupsList();
+
+            group._groups = new FasterList<ExclusiveGroupStruct>();
+            group._sets   = new HashSet<ExclusiveGroupStruct>();
 
-        static readonly FasterList<ExclusiveGroupStruct> groups;
-        static readonly HashSet<ExclusiveGroupStruct>    sets;
+            return group;
+        }
 
-        public void Reset() { sets.Clear(); }
+        public void Reset()
+        {
+            _sets.Clear();
+        }
 
         public void AddRange(ExclusiveGroupStruct[] groupsToAdd, int length)
         {
-            for (int i = 0; i < length; i++)
-            {
-                sets.Add(groupsToAdd[i]);
-            }
+            for (var i = 0; i < length; i++) _sets.Add(groupsToAdd[i]);
         }
 
-        public void Add(ExclusiveGroupStruct @group) { sets.Add(group); }
+        public void Add(ExclusiveGroupStruct group)
+        {
+            _sets.Add(group);
+        }
 
         public void Exclude(ExclusiveGroupStruct[] groupsToIgnore, int length)
         {
-            for (int i = 0; i < length; i++)
-            {
-                sets.Remove(groupsToIgnore[i]);
-            }
+            for (var i = 0; i < length; i++) _sets.Remove(groupsToIgnore[i]);
         }
 
-        public void Exclude(ExclusiveGroupStruct groupsToIgnore) { sets.Remove(groupsToIgnore); }
+        public void Exclude(ExclusiveGroupStruct groupsToIgnore)
+        {
+            _sets.Remove(groupsToIgnore);
+        }
 
-        public void EnsureCapacity(uint preparecount) { groups.EnsureCapacity(preparecount); }
+        public void Resize(uint preparecount)
+        {
+            _groups.Resize(preparecount);
+        }
 
         public FasterList<ExclusiveGroupStruct> Evaluate()
         {
-            groups.FastClear();
+            _groups.FastClear();
 
-            foreach (var item in sets)
-            {
-                groups.Add(item);
-            }
+            foreach (var item in _sets) _groups.Add(item);
 
-            return groups;
+            return _groups;
         }
+        
+        FasterList<ExclusiveGroupStruct> _groups;
+        HashSet<ExclusiveGroupStruct>    _sets;
     }
 
+    //I am not 100% sure why I made this thread-safe since it cannot be used inside jobs.
     public ref struct QueryGroups
     {
-        static readonly ThreadLocal<GroupsList> groups = new ThreadLocal<GroupsList>();
+        static readonly ThreadLocal<GroupsList> groups;
+
+        static QueryGroups()
+        {
+            groups = new ThreadLocal<GroupsList>(GroupsList.Init);
+        }
 
         public QueryGroups(LocalFasterReadOnlyList<ExclusiveGroupStruct> groups)
         {
             var groupsValue = QueryGroups.groups.Value;
 
             groupsValue.Reset();
-             groupsValue.AddRange(groups.ToArrayFast(out var count), count);
+            groupsValue.AddRange(groups.ToArrayFast(out var count), count);
         }
 
-        public QueryGroups(ExclusiveGroupStruct @group)
+        public QueryGroups(ExclusiveGroupStruct group)
         {
             var groupsValue = groups.Value;
 
             groupsValue.Reset();
-            groupsValue.Add(@group);
+            groupsValue.Add(group);
         }
 
         public QueryGroups(uint preparecount)
@@ -78,7 +91,7 @@ namespace Svelto.ECS.Experimental
             var groupsValue = groups.Value;
 
             groupsValue.Reset();
-            groupsValue.EnsureCapacity(preparecount);
+            groupsValue.Resize(preparecount);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -114,7 +127,7 @@ namespace Svelto.ECS.Experimental
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public QueryGroups Except(ExclusiveGroupStruct[] groupsToIgnore)
         {
-            var groupsValue = QueryGroups.groups.Value;
+            var groupsValue = groups.Value;
 
             groupsValue.Exclude(groupsToIgnore, groupsToIgnore.Length);
 
@@ -124,7 +137,7 @@ namespace Svelto.ECS.Experimental
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public QueryGroups Except(LocalFasterReadOnlyList<ExclusiveGroupStruct> groupsToIgnore)
         {
-            var groupsValue = QueryGroups.groups.Value;
+            var groupsValue = groups.Value;
 
             groupsValue.Exclude(groupsToIgnore.ToArrayFast(out var count), count);
 
@@ -134,7 +147,7 @@ namespace Svelto.ECS.Experimental
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public QueryGroups Except(FasterList<ExclusiveGroupStruct> groupsToIgnore)
         {
-            var groupsValue = QueryGroups.groups.Value;
+            var groupsValue = groups.Value;
 
             groupsValue.Exclude(groupsToIgnore.ToArrayFast(out var count), count);
 
@@ -144,7 +157,7 @@ namespace Svelto.ECS.Experimental
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public QueryGroups Except(FasterReadOnlyList<ExclusiveGroupStruct> groupsToIgnore)
         {
-            var groupsValue = QueryGroups.groups.Value;
+            var groupsValue = groups.Value;
 
             groupsValue.Exclude(groupsToIgnore.ToArrayFast(out var count), count);
 
@@ -176,29 +189,49 @@ namespace Svelto.ECS.Experimental
 
             return new QueryResult(groupsValue.Evaluate());
         }
+        
+        public void Evaluate(FasterList<ExclusiveGroupStruct> group)
+        {
+            var groupsValue = groups.Value;
+
+            groupsValue.Evaluate().CopyTo(group.ToArrayFast(out var count), count);
+        }
     }
 
     public readonly ref struct QueryResult
     {
-        public QueryResult(FasterList<ExclusiveGroupStruct> @group) { _group = @group; }
+        public QueryResult(FasterList<ExclusiveGroupStruct> group)
+        {
+            _group = group;
+        }
 
         public LocalFasterReadOnlyList<ExclusiveGroupStruct> result => _group;
 
         readonly FasterReadOnlyList<ExclusiveGroupStruct> _group;
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public int Count<T>(EntitiesDB entitiesDB)
-            where T : struct, IEntityComponent
+        public int Count<T>(EntitiesDB entitiesDB) where T : struct, IEntityComponent
+        {
+            var count = 0;
+
+            var groupsCount                             = result.count;
+            for (var i = 0; i < groupsCount; ++i) count += entitiesDB.Count<T>(result[i]);
+
+            return count;
+        }
+
+        public int Max<T>(EntitiesDB entitiesDB) where T : struct, IEntityComponent
         {
-            int count = 0;
+            var max = 0;
 
-            var groupsCount = result.count;
-            for (int i = 0; i < groupsCount; ++i)
+            var groupsCount                           = result.count;
+            for (var i = 0; i < groupsCount; ++i)
             {
-                count += entitiesDB.Count<T>(result[i]);
+                var count = entitiesDB.Count<T>(result[i]);
+                if (count > max) max = count;
             }
 
-            return count;
+            return max;
         }
     }
 }

Core/ReactEngineContainer.cs

@@ -2,12 +2,12 @@ using Svelto.ECS.Internal;
 
 namespace Svelto.ECS
 {
-    public readonly struct ReactEngineContainer
+    public readonly struct ReactEngineContainer<T> where T:IReactEngine
     {
         public readonly string name;
-        public readonly         IReactEngine engine;
+        public readonly T      engine;
 
-        public ReactEngineContainer(IReactEngine engine, string name)
+        public ReactEngineContainer(T engine, string name)
         {
             this.name   = name;
             this.engine = engine;

Core/SetEGIDWithoutBoxing.cs

@@ -1,5 +1,4 @@
-using Svelto.ECS.Reference;
-
+#if SLOW_SVELTO_SUBMISSION
 namespace Svelto.ECS.Internal
 {
     delegate void SetEGIDWithoutBoxingActionCast<T>(ref T target, EGID egid) where T : struct, IEntityComponent;
@@ -67,4 +66,5 @@ namespace Svelto.ECS.Internal
             }
         }
     }
-}
+}
+#endif

Core/SimpleEntitiesSubmissionScheduler.cs

@@ -1,60 +1,33 @@
-using System;
-using System.Collections.Generic;
-
-namespace Svelto.ECS.Schedulers
+namespace Svelto.ECS.Schedulers
 {
     public sealed class SimpleEntitiesSubmissionScheduler : EntitiesSubmissionScheduler
     {
-        public SimpleEntitiesSubmissionScheduler(uint maxNumberOfOperationsPerFrame = UInt32.MaxValue)
-        {
-            _enumerator = SubmitEntitiesAsync(maxNumberOfOperationsPerFrame);
-        }
-
-        public IEnumerator<bool> SubmitEntitiesAsync()
-        {
-            return _enumerator;
-        }
-
-        public IEnumerator<bool> SubmitEntitiesAsync(uint maxNumberOfOperations)
+        protected internal override EnginesRoot.EntitiesSubmitter onTick
         {
-            EnginesRoot.EntitiesSubmitter entitiesSubmitter = _onTick.Value;
-            entitiesSubmitter.maxNumberOfOperationsPerFrame = maxNumberOfOperations;
-
-            while (true)
+            set
             {
-                if (paused == false)
-                {
-                    var entitiesSubmitterSubmitEntities = entitiesSubmitter.submitEntities;
-                    
-                    entitiesSubmitterSubmitEntities.MoveNext();
-                    
-                    yield return entitiesSubmitterSubmitEntities.Current == true;
-                }
-            }
-        }
+                DBC.ECS.Check.Require(_entitiesSubmitter == null, "a scheduler can be exclusively used by one enginesRoot only");
 
-        public void SubmitEntities()
-        {
-            do
-            {
-                _enumerator.MoveNext();
-            } while (_enumerator.Current == true);
+                _entitiesSubmitter = value;
+            }
         }
 
-        public override bool paused    { get; set; }
         public override void Dispose() { }
 
-        protected internal override EnginesRoot.EntitiesSubmitter onTick
+        public void SubmitEntities()
         {
-            set
+            try
             {
-                DBC.ECS.Check.Require(_onTick == null, "a scheduler can be exclusively used by one enginesRoot only");
-
-                _onTick = value;
+                _entitiesSubmitter.Value.SubmitEntities();
+            }
+            catch
+            {
+                paused = true;
+                
+                throw;
             }
         }
 
-        EnginesRoot.EntitiesSubmitter? _onTick;
-        readonly IEnumerator<bool>     _enumerator;
+        EnginesRoot.EntitiesSubmitter? _entitiesSubmitter;
     }
 }

Core/SpecialEnumerators/WaitForSubmissionEnumerator.cs

@@ -7,9 +7,9 @@ namespace Svelto.ECS
     /// <summary>
     /// Enumerator that yields until the next Entities Submission
     /// </summary>
-    public struct WaitForSubmissionEnumerator : IEnumerator
+    public class WaitForSubmissionEnumerator : IEnumerator
     {
-        public WaitForSubmissionEnumerator(EntitiesSubmissionScheduler scheduler):this()
+        public WaitForSubmissionEnumerator(EntitiesSubmissionScheduler scheduler)
         {
             _scheduler       = scheduler;
         }

Core/Streams/EntitiesStreams.cs

@@ -4,15 +4,13 @@ using Svelto.DataStructures;
 namespace Svelto.ECS
 {
     /// <summary>
-    ///     I eventually realised that, with the ECS design, no form of communication other than polling entity components can
-    ///     exist.
-    ///     Using groups, you can have always an optimal set of entity components to poll. However EntityStreams
-    ///     can be useful if:
-    ///     - you need to react on seldom entity changes, usually due to user events
-    ///     - you want engines to be able to track entity changes
-    ///     - you want a thread-safe way to read entity states, which includes all the state changes and not the last
-    ///     one only
-    ///     - you want to communicate between EnginesRoots
+    ///     I eventually realised that, with the ECS design, no form of engines (systems) communication other
+    ///     than polling entity components is effective.
+    ///     The only purpose of this publisher/consumer model is to let two enginesroots communicate with each other
+    ///     through a thread safe ring buffer.
+    ///     The engines root A publishes entities.
+    ///     The engines root B can consume those entities at any time, as they will be a copy of the original
+    ///     entities and won't point directly to the database of the engines root A
     /// </summary>
     struct EntitiesStreams : IDisposable
     {
@@ -46,7 +44,7 @@ namespace Svelto.ECS
         public void Dispose()
         {
             foreach (var stream in _streams)
-                stream.Value.Dispose();
+                stream.value.Dispose();
         }
 
         public static EntitiesStreams Create()

Core/Streams/ThreadSafeNativeEntityStream.cs

@@ -1,31 +0,0 @@
-namespace Svelto.ECS
-{
-    /// <summary>
-    /// This EntityStream can be used in parallel jobs, but does NOT guarantee order.
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-    public struct ThreadSafeNativeEntityStream<T> : ITypeSafeStream
-    {
-        public ThreadSafeNativeEntityStream(EntitiesDB entitiesDB)
-        {
-        }
-
-        public void Dispose()
-        {
-            
-        }
-
-        /// <summary>
-        /// I am thinking to pass the component and do the queryEntity only as a validation
-        /// </summary>
-        /// <param name="entityComponent"></param>
-        /// <param name="id"></param>
-        public void PublishEntityChange(in T entityComponent, EGID id)
-        {
-#if DEBUG && !PROFILE_SVELTO
-            
-#endif            
-            
-        }
-    }
-}

Core/Streams/ThreadSafeNativeEntityStream.cs.meta

@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: a0c49b065ae330f083a88e1b042a108a
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 

DataStructures/ITypeSafeDictionary.cs

@@ -7,37 +7,80 @@ namespace Svelto.ECS.Internal
     public interface ITypeSafeDictionary<TValue> : ITypeSafeDictionary where TValue : IEntityComponent
     {
         void Add(uint egidEntityId, in TValue entityComponent);
-        ref TValue this[uint idEntityId] { get; }
-        bool TryGetValue(uint entityId, out TValue item);
-        ref TValue GetOrCreate(uint idEntityId);
+        
+        bool       TryGetValue(uint entityId, out TValue item);
+        ref TValue GetOrAdd(uint idEntityId);
 
         IBuffer<TValue> GetValues(out uint count);
-        ref TValue GetDirectValueByRef(uint key);
+        ref TValue      GetDirectValueByRef(uint key);
+        ref TValue      GetValueByRef(uint key);
+        EntityIDs       entityIDs { get; }
     }
 
-    public interface ITypeSafeDictionary:IDisposable
+    public interface ITypeSafeDictionary : IDisposable
     {
-        uint                count { get; }
+        int                 count { get; }
+        
         ITypeSafeDictionary Create();
 
-        //todo: there is something wrong in the design of the execute callback methods. Something to cleanup
-        void ExecuteEnginesAddOrSwapCallbacks(FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> entityComponentEnginesDb,
-            ITypeSafeDictionary realDic, ExclusiveGroupStruct? fromGroup, ExclusiveGroupStruct toGroup, in PlatformProfiler profiler);
-        void ExecuteEnginesSwapOrRemoveCallbacks(EGID fromEntityGid, EGID? toEntityID, ITypeSafeDictionary toGroup,
-                                                 FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> engines, in PlatformProfiler profiler);
-        void ExecuteEnginesRemoveCallbacks(FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer>> entityComponentEnginesDB,
-            in PlatformProfiler profiler, ExclusiveGroupStruct @group);
+        void AddEntitiesToDictionary
+        (ITypeSafeDictionary toDictionary, ExclusiveGroupStruct groupId, in EnginesRoot.EntityReferenceMap entityLocator);
+        void RemoveEntitiesFromDictionary(FasterList<(uint, string)> infosToProcess);
+        void SwapEntitiesBetweenDictionaries(FasterList<(uint, uint, string)> infosToProcess,
+         ExclusiveGroupStruct fromGroup, ExclusiveGroupStruct toGroup, ITypeSafeDictionary toComponentsDictionary);
+        
+        //------------
+
+        //This is now obsolete, but I cannot mark it as such because it's heavily used by legacy projects
+        void ExecuteEnginesAddCallbacks
+        (FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnAdd>>> entityComponentEnginesDb
+       , ITypeSafeDictionary destinationDatabase, ExclusiveGroupStruct toGroup, in PlatformProfiler profiler);
+        //Version to use
+        void ExecuteEnginesAddEntityCallbacksFast(
+         FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnAddEx>>> reactiveEnginesAdd,
+         ExclusiveGroupStruct groupID, (uint, uint) rangeOfSubmittedEntitiesIndicies, in PlatformProfiler profiler);
+
+        //------------
+        
+        //This is now obsolete, but I cannot mark it as such because it's heavily used by legacy projects
+        void ExecuteEnginesSwapCallbacks(FasterList<(uint, uint, string)> infosToProcess,
+         FasterList<ReactEngineContainer<IReactOnSwap>> reactiveEnginesSwap, ExclusiveGroupStruct fromGroup,
+         ExclusiveGroupStruct toGroup, in PlatformProfiler sampler);
+        //Version to use
+        void ExecuteEnginesSwapCallbacksFast(FasterList<ReactEngineContainer<IReactOnSwapEx>> reactiveEnginesSwap,
+         ExclusiveGroupStruct fromGroup, ExclusiveGroupStruct toGroup, (uint, uint) rangeOfSubmittedEntitiesIndicies,
+         in PlatformProfiler sampler);
         
-        void AddEntitiesFromDictionary
-            (ITypeSafeDictionary entitiesToSubmit, ExclusiveGroupStruct groupId, EnginesRoot enginesRoot);
+        //------------
         
-        void AddEntityToDictionary(EGID fromEntityGid, EGID toEntityID, ITypeSafeDictionary toGroup);
-        void RemoveEntityFromDictionary(EGID fromEntityGid);
+        //This is now obsolete, but I cannot mark it as such because it's heavily used by legacy projects
+        void ExecuteEnginesRemoveCallbacks(FasterList<(uint, string)> infosToProcess,
+         FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnRemove>>> reactiveEnginesRemove,
+         ExclusiveGroupStruct fromGroup, in PlatformProfiler sampler);
+        //Version to use
+        void ExecuteEnginesRemoveCallbacksFast(FasterList<ReactEngineContainer<IReactOnRemoveEx>> reactiveEnginesRemoveEx,
+         ExclusiveGroupStruct fromGroup, (uint, uint) rangeOfSubmittedEntitiesIndicies,
+         in PlatformProfiler sampler);
+        
+        //------------
+
+        void ExecuteEnginesSwapCallbacks_Group(
+         FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnSwap>>> reactiveEnginesSwap,
+         FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnSwapEx>>> reactiveEnginesSwapEx,
+         ITypeSafeDictionary toEntitiesDictionary, ExclusiveGroupStruct fromGroupId, ExclusiveGroupStruct toGroupId,
+         in PlatformProfiler platformProfiler);
+        void ExecuteEnginesRemoveCallbacks_Group(
+         FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnRemove>>> engines,
+         FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnRemoveEx>>> reactiveEnginesRemoveEx,
+         ExclusiveGroupStruct @group, in PlatformProfiler profiler);
+        void ExecuteEnginesDisposeCallbacks_Group
+        (FasterDictionary<RefWrapperType, FasterList<ReactEngineContainer<IReactOnDispose>>> engines
+       , ExclusiveGroupStruct group, in PlatformProfiler profiler);
 
-        void ResizeTo(uint size);
+        void IncreaseCapacityBy(uint size);
+        void EnsureCapacity(uint size);
         void Trim();
         void Clear();
-        void FastClear();
         bool Has(uint key);
         bool ContainsKey(uint egidEntityId);
         uint GetIndex(uint valueEntityId);

DataStructures/Unmanaged/AtomicNativeBags.cs

@@ -75,7 +75,7 @@ namespace Svelto.ECS.DataStructures
         readonly Allocator _allocator;
         readonly uint      _threadsCount;
         
-#if UNITY_COLLECTIONS || UNITY_JOBS || UNITY_BURST    
+#if UNITY_COLLECTIONS || UNITY_JOBS || UNITY_BURST
 #if UNITY_BURST
         [Unity.Burst.NoAlias]
 #endif

DataStructures/Unmanaged/NativeBag.cs

@@ -1,15 +1,11 @@
 #if DEBUG && !PROFILE_SVELTO
 #define ENABLE_DEBUG_CHECKS
 #endif
-
-#if DEBUG && !PROFILE_SVELTO
-//#define ENABLE_THREAD_SAFE_CHECKS
-#endif
-
 using System;
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using Svelto.Common;
+using Svelto.Common.DataStructures;
 
 namespace Svelto.ECS.DataStructures
 {
@@ -34,18 +30,11 @@ namespace Svelto.ECS.DataStructures
                 unsafe
                 {
                     BasicTests();
-#if ENABLE_THREAD_SAFE_CHECKS
-                    try
+                    
+                    using (_threadSentinel.TestThreadSafety())
                     {
-#endif
-                    return _queue->size;
-#if ENABLE_THREAD_SAFE_CHECKS
-                    }
-                    finally
-                    {
-                        Volatile.Write(ref _threadSentinel, 0);
+                        return _queue->size;
                     }
-#endif
                 }
             }
         }
@@ -58,35 +47,23 @@ namespace Svelto.ECS.DataStructures
                 unsafe
                 {
                     BasicTests();
-#if ENABLE_THREAD_SAFE_CHECKS
-                    try
+                    
+                    using (_threadSentinel.TestThreadSafety())
                     {
-#endif
-                    return _queue->capacity;
-#if ENABLE_THREAD_SAFE_CHECKS
-                    }
-                    finally
-                    {
-                        Volatile.Write(ref _threadSentinel, 0);
+                        return _queue->capacity;
                     }
-#endif
                 }
             }
         }
 
-        public NativeBag(Allocator allocator)
+        public NativeBag(Allocator allocator):this()
         {
             unsafe
             {
-                var listData = (UnsafeBlob*) MemoryUtilities.Alloc<UnsafeBlob>((uint) 1, allocator);
+                var listData = (UnsafeBlob*)MemoryUtilities.Alloc<UnsafeBlob>((uint)1, allocator);
 
-                //clear to nullify the pointers
-                //MemoryUtilities.MemClear((IntPtr) listData, (uint) sizeOf);
                 listData->allocator = allocator;
                 _queue              = listData;
-#if ENABLE_THREAD_SAFE_CHECKS
-                _threadSentinel = 0;
-#endif
             }
         }
 
@@ -96,19 +73,12 @@ namespace Svelto.ECS.DataStructures
             unsafe
             {
                 BasicTests();
-#if ENABLE_THREAD_SAFE_CHECKS
-                try
-                {
-#endif
-                if (_queue == null || _queue->ptr == null)
-                    return true;
-#if ENABLE_THREAD_SAFE_CHECKS
-                }
-                finally
+
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    Volatile.Write(ref _threadSentinel, 0);
+                    if (_queue == null || _queue->ptr == null)
+                        return true;
                 }
-#endif
             }
 
             return count == 0;
@@ -119,84 +89,58 @@ namespace Svelto.ECS.DataStructures
         {
             if (_queue != null)
             {
-#if ENABLE_THREAD_SAFE_CHECKS
-                //todo: this must be unit tested
-                if (Interlocked.CompareExchange(ref _threadSentinel, 1, 0) != 0)
-                    throw new Exception("NativeBag is not thread safe, reading and writing operations can happen" +
-                        "on different threads, but not simultaneously");
+                BasicTests();
 
-                try
+                using (_threadSentinel.TestThreadSafety())
                 {
-#endif
-                _queue->Dispose();
-                MemoryUtilities.Free((IntPtr) _queue, _queue->allocator);
-                _queue = null;
-#if ENABLE_THREAD_SAFE_CHECKS
-                }
-                finally
-                {
-                Volatile.Write(ref _threadSentinel, 0);
+                    _queue->Dispose();
+                    MemoryUtilities.Free((IntPtr)_queue, _queue->allocator);
+                    _queue = null;
                 }
-#endif
             }
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public ref T ReserveEnqueue<T>(out UnsafeArrayIndex index) where T : struct
+        public ref T ReserveEnqueue<T>
+            (out UnsafeArrayIndex index)
+            where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
                 BasicTests();
 
                 var sizeOf = MemoryUtilities.SizeOf<T>();
-                if (_queue->availableSpace - sizeOf < 0)
+                
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    _queue->Realloc((_queue->capacity + (uint)sizeOf) << 1);
-                }
-
-#if ENABLE_THREAD_SAFE_CHECKS
-                try
-                {
-#endif
+                    if (_queue->availableSpace - sizeOf < 0)
+                    {
+                        _queue->Grow<T>();
+                    }
 
-                return ref _queue->Reserve<T>(out index);
-#if ENABLE_THREAD_SAFE_CHECKS
+                    return ref _queue->Reserve<T>(out index);
                 }
-                finally
-                {
-                    Volatile.Write(ref _threadSentinel, 0);
-                }
-#endif
             }
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void Enqueue<T>(in T item) where T : struct
+        public void Enqueue<T>
+            (in T item) where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
                 BasicTests();
 
-#if ENABLE_THREAD_SAFE_CHECKS
-                try
-                {
-#endif
-                var sizeOf = MemoryUtilities.SizeOf<T>();
-                if (_queue->availableSpace - sizeOf < 0)
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    var capacityInBytes = (_queue->capacity + (uint)sizeOf);
-
-                    _queue->Realloc(capacityInBytes << 1);
-                }
+                    var sizeOf = MemoryUtilities.SizeOf<T>();
+                    if (_queue->availableSpace - sizeOf < 0)
+                    {
+                        _queue->Grow<T>();
+                    }
 
-                _queue->Enqueue(item);
-#if ENABLE_THREAD_SAFE_CHECKS
-                }
-                finally
-                {
-                Volatile.Write(ref _threadSentinel, 0);
+                    _queue->Enqueue(item);
                 }
-#endif
             }
         }
 
@@ -206,58 +150,37 @@ namespace Svelto.ECS.DataStructures
             unsafe
             {
                 BasicTests();
-#if ENABLE_THREAD_SAFE_CHECKS
-                try
-                {
-#endif
-                _queue->Clear();
-#if ENABLE_THREAD_SAFE_CHECKS
-                }
-                finally
+
+                using (_threadSentinel.TestThreadSafety())
                 {
-                Volatile.Write(ref _threadSentinel, 0);
+                    _queue->Clear();
                 }
-#endif
             }
         }
 
-        public T Dequeue<T>() where T : struct
+        public T Dequeue<T>() where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
                 BasicTests();
-#if ENABLE_THREAD_SAFE_CHECKS
-                try
-                {
-#endif
-                return _queue->Dequeue<T>();
-#if ENABLE_THREAD_SAFE_CHECKS
-                }
-                finally
+
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    Volatile.Write(ref _threadSentinel, 0);
+                    return _queue->Dequeue<T>();
                 }
-#endif
             }
         }
 
-        public ref T AccessReserved<T>(UnsafeArrayIndex reservedIndex) where T : struct
+        public ref T AccessReserved<T>(UnsafeArrayIndex reservedIndex) where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
                 BasicTests();
-#if ENABLE_THREAD_SAFE_CHECKS
-                try
-                {
-#endif
-                return ref _queue->AccessReserved<T>(reservedIndex);
-#if ENABLE_THREAD_SAFE_CHECKS
-                }
-                finally
+
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    Volatile.Write(ref _threadSentinel, 0);
+                    return ref _queue->AccessReserved<T>(reservedIndex);
                 }
-#endif
             }
         }
 
@@ -266,17 +189,10 @@ namespace Svelto.ECS.DataStructures
         {
             if (_queue == null)
                 throw new Exception("SimpleNativeArray: null-access");
-#if ENABLE_THREAD_SAFE_CHECKS
-            todo: this must be unit tested
-             if (Interlocked.CompareExchange(ref _threadSentinel, 1, 0) != 0)
-                 throw new Exception("NativeBag is not thread safe, reading and writing operations can happen"
-                                   + "on different threads, but not simultaneously");
-#endif
         }
+        
+        readonly Sentinel _threadSentinel;
 
-#if ENABLE_THREAD_SAFE_CHECKS
-        int _threadSentinel;
-#endif
 #if UNITY_COLLECTIONS || UNITY_JOBS || UNITY_BURST
 #if UNITY_BURST
         [Unity.Burst.NoAlias]

DataStructures/Unmanaged/NativeDynamicArray.cs

@@ -1,6 +1,11 @@
+#if DEBUG && !PROFILE_SVELTO
+#define ENABLE_DEBUG_CHECKS
+#endif
+
 using System;
 using System.Runtime.CompilerServices;
 using Svelto.Common;
+using Svelto.Common.DataStructures;
 using Allocator = Svelto.Common.Allocator;
 
 namespace Svelto.ECS.DataStructures
@@ -24,7 +29,7 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
@@ -34,13 +39,13 @@ namespace Svelto.ECS.DataStructures
                 return (_list->count / MemoryUtilities.SizeOf<T>());
             }
         }
-        
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public int Size() 
+        public int SizeInBytes()
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
 
@@ -54,7 +59,7 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
@@ -74,16 +79,19 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
-                var rtnStruc = new NativeDynamicArray {_hashType = TypeHash<T>.hash};
-#else           
+#if ENABLE_DEBUG_CHECKS
+                var rtnStruc = new NativeDynamicArray
+                {
+                    _hashType       = TypeHash<T>.hash,
+                };
+#else
                 NativeDynamicArray rtnStruc = default;
 #endif
-                UnsafeArray* listData    = (UnsafeArray*) MemoryUtilities.Alloc<UnsafeArray>(1, allocator);
+                UnsafeArray* listData = (UnsafeArray*)MemoryUtilities.Alloc<UnsafeArray>(1, allocator);
 
                 //clear to nullify the pointers
                 //MemoryUtilities.MemClear((IntPtr) listData, structSize);
-
+                
                 rtnStruc._allocator = allocator;
                 listData->Realloc<T>(newLength, allocator);
 
@@ -98,7 +106,7 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
@@ -106,14 +114,22 @@ namespace Svelto.ECS.DataStructures
                 if (index >= Count<T>())
                     throw new Exception($"NativeDynamicArray: out of bound access, index {index} count {Count<T>()}");
 #endif
-                return ref _list->Get<T>(index);
+                
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    return ref _list->Get<T>(index);
+#if ENABLE_DEBUG_CHECKS                    
+                }
+#endif
             }
         }
-        
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public ref T Get<T>(int index) where T : struct
         {
-            return ref Get<T>((uint) index);
+            return ref Get<T>((uint)index);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -121,27 +137,45 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
                 if (index >= Capacity<T>())
-                    throw new Exception($"NativeDynamicArray: out of bound access, index {index} capacity {Capacity<T>()}");
+                    throw new Exception(
+                        $"NativeDynamicArray: out of bound access, index {index} capacity {Capacity<T>()}");
+#endif
+                
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    _list->Set(index, value);
+                    
+#if ENABLE_DEBUG_CHECKS
+                }
 #endif
-                _list->Set(index, value);
             }
         }
 
         public unsafe void Dispose()
         {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
             if (_list == null)
                 throw new Exception("NativeDynamicArray: null-access");
 #endif
-            _list->Dispose(_allocator);
-            MemoryUtilities.Free((IntPtr) _list, _allocator);
             
+#if ENABLE_DEBUG_CHECKS
+            using (_threadSentinel.TestThreadSafety())
+            {
+#endif
+                _list->Dispose(_allocator);
+                MemoryUtilities.Free((IntPtr)_list, _allocator);
+                
+#if ENABLE_DEBUG_CHECKS
+            }
+#endif
             _list = null;
         }
 
@@ -150,56 +184,80 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
 #endif
-                if (Count<T>() == Capacity<T>())
+                
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    _list->Realloc<T>((uint) ((Capacity<T>() + 1) * 1.5f), _allocator);
-                }
+#endif
+                    if (Count<T>() == Capacity<T>())
+                    {
+                        _list->Realloc<T>((uint)((Capacity<T>() + 1) * 1.5f), _allocator);
+                    }
 
-                _list->Add(item);
+                    _list->Add(item);
+#if ENABLE_DEBUG_CHECKS                    
+                }
+#endif
             }
         }
-        
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public ref T AddAt<T>(uint index) where T : struct
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
 #endif
-                var structSize = (uint) MemoryUtilities.SizeOf<T>();
+                var structSize = (uint)MemoryUtilities.SizeOf<T>();
 
-                if (index >= Capacity<T>())
-                    _list->Realloc<T>((uint) ((index + 1) * 1.5f), _allocator);
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    if (index >= Capacity<T>())
+                        _list->Realloc<T>((uint)((index + 1) * 1.5f), _allocator);
 
-                var writeIndex = (index + 1) * structSize;
-                if (_list->count < writeIndex)
-                    _list->SetCountTo(writeIndex);
+                    var writeIndex = (index + 1) * structSize;
+                    if (_list->count < writeIndex)
+                        _list->SetCountTo(writeIndex);
 
-                return ref _list->Get<T>(index);
+                    return ref _list->Get<T>(index);
+#if ENABLE_DEBUG_CHECKS                    
+                }
+#endif
             }
         }
-        
+
         public void Resize<T>(uint newCapacity) where T : struct
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
 #endif
-                _list->Realloc<T>((uint) newCapacity, _allocator);
+                
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    _list->Realloc<T>((uint)newCapacity, _allocator);
+                    
+#if ENABLE_DEBUG_CHECKS
+                }
+#endif
             }
         }
 
@@ -207,16 +265,24 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
 #endif
-                uint structSize = (uint) MemoryUtilities.SizeOf<T>();
-                uint size       = (uint) (count * structSize);
+                uint structSize = (uint)MemoryUtilities.SizeOf<T>();
+                uint size       = (uint)(count * structSize);
 
-                _list->SetCountTo((uint) size);
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    _list->SetCountTo((uint)size);
+                    
+#if ENABLE_DEBUG_CHECKS
+                }
+#endif
             }
         }
 
@@ -225,18 +291,27 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
 
-                var structSize = (uint) MemoryUtilities.SizeOf<T>();
-                
+                var structSize = (uint)MemoryUtilities.SizeOf<T>();
+
                 if (_list->space - (int)structSize < 0)
                     throw new Exception("NativeDynamicArray: no writing authorized");
 #endif
-                _list->Add(item);
+                
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    _list->Add(item);
+                    
+#if ENABLE_DEBUG_CHECKS
+                }
+#endif
             }
         }
 
@@ -245,7 +320,7 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
@@ -253,13 +328,22 @@ namespace Svelto.ECS.DataStructures
                 if (Count<T>() == 0)
                     throw new Exception("NativeDynamicArray: empty array invalid operation");
 #endif
-                var indexToMove = Count<T>() - 1;
-                if (index < indexToMove)
+
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    Set<T>(index, Get<T>((uint) indexToMove));
+#endif
+                    var indexToMove = Count<T>() - 1;
+                    if (index < indexToMove)
+                    {
+                        Set<T>(index, Get<T>((uint)indexToMove));
+                    }
+
+                    _list->Pop<T>();
+                    
+#if ENABLE_DEBUG_CHECKS
                 }
-
-                _list->Pop<T>();
+#endif
             }
         }
 
@@ -268,38 +352,47 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
 #endif
-                _list->Clear();
+                
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    _list->Clear();
+                    
+#if ENABLE_DEBUG_CHECKS
+                }
+#endif
             }
         }
 
         public unsafe T* ToPTR<T>() where T : unmanaged
         {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
             if (_list == null)
                 throw new Exception("NativeDynamicArray: null-access");
             if (_hashType != TypeHash<T>.hash)
                 throw new Exception("NativeDynamicArray: not expected type used");
 
 #endif
-            return (T*) _list->ptr;
+            return (T*)_list->ptr;
         }
 
         public IntPtr ToIntPTR<T>() where T : struct
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
-            if (_list == null)
-                throw new Exception("NativeDynamicArray: null-access");
-            if (_hashType != TypeHash<T>.hash)
-                throw new Exception("NativeDynamicArray: not expected type used");
+#if ENABLE_DEBUG_CHECKS
+                if (_list == null)
+                    throw new Exception("NativeDynamicArray: null-access");
+                if (_hashType != TypeHash<T>.hash)
+                    throw new Exception("NativeDynamicArray: not expected type used");
 
 #endif
-                return (IntPtr) _list->ptr;
+                return (IntPtr)_list->ptr;
             }
         }
 
@@ -307,7 +400,7 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
@@ -318,12 +411,20 @@ namespace Svelto.ECS.DataStructures
                 var ret                 = new T[count];
                 var lengthToCopyInBytes = count * MemoryUtilities.SizeOf<T>();
 
-                fixed (void* handle = ret)
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    Unsafe.CopyBlock(handle, _list->ptr, (uint) lengthToCopyInBytes);
+#endif
+                    fixed (void* handle = ret)
+                    {
+                        Unsafe.CopyBlock(handle, _list->ptr, (uint)lengthToCopyInBytes);
+                    }
+
+                    return ret;
+                    
+#if ENABLE_DEBUG_CHECKS
                 }
-
-                return ret;
+#endif
             }
         }
 
@@ -331,20 +432,27 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
 #endif
-                var capacity            = Capacity<T>();
-                var lengthToCopyInBytes = capacity * MemoryUtilities.SizeOf<T>();
-                var ret                 = new T[capacity];
+                var capacity = Capacity<T>();
+                var ret      = new T[capacity];
 
-                fixed (void* handle = ret)
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
                 {
-                    Unsafe.CopyBlock(handle, _list->ptr, (uint) lengthToCopyInBytes);
+#endif
+                    fixed (void* handle = ret)
+                    {
+                        MemoryUtilities.MemCpy<T>((IntPtr)_list->ptr, 0, (IntPtr)handle, 0, (uint)capacity);
+                    }
+                    
+#if ENABLE_DEBUG_CHECKS
                 }
+#endif
 
                 return ret;
             }
@@ -354,18 +462,24 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-#if DEBUG && !PROFILE_SVELTO
+#if ENABLE_DEBUG_CHECKS
                 if (_list == null)
                     throw new Exception("NativeDynamicArray: null-access");
                 if (_hashType != TypeHash<T>.hash)
                     throw new Exception("NativeDynamicArray: not expected type used");
 #endif
 
-                var sizeOf = MemoryUtilities.SizeOf<T>();
-                //Unsafe.CopyBlock may not be memory overlapping safe (memcpy vs memmove)
-                Buffer.MemoryCopy(_list->ptr + (index + 1) * sizeOf, _list->ptr + index * sizeOf, _list->count
-                                , (uint) ((Count<T>() - (index + 1)) * sizeOf));
-                _list->Pop<T>();
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    MemoryUtilities.MemMove<T>((IntPtr)_list->ptr, index + 1, index, (uint)(Count<T>() - (index + 1)));
+
+                    _list->Pop<T>();
+                    
+#if ENABLE_DEBUG_CHECKS
+                }
+#endif
             }
         }
 
@@ -373,11 +487,18 @@ namespace Svelto.ECS.DataStructures
         {
             unsafe
             {
-                MemoryUtilities.MemClear((IntPtr) _list->ptr, (uint) _list->capacity);
+#if ENABLE_DEBUG_CHECKS
+                using (_threadSentinel.TestThreadSafety())
+                {
+#endif
+                    MemoryUtilities.MemClear((IntPtr)_list->ptr, (uint)_list->capacity);
+#if ENABLE_DEBUG_CHECKS
+                }
+#endif
             }
         }
-        
-#if UNITY_COLLECTIONS || UNITY_JOBS || UNITY_BURST    
+
+#if UNITY_COLLECTIONS || UNITY_JOBS || UNITY_BURST
 #if UNITY_BURST
         [Unity.Burst.NoAlias]
 #endif
@@ -387,6 +508,9 @@ namespace Svelto.ECS.DataStructures
 #if DEBUG && !PROFILE_SVELTO
         int _hashType;
 #endif
+        
+        Sentinel _threadSentinel;
+
         Allocator _allocator;
     }
 }

DataStructures/Unmanaged/NativeDynamicArrayCast.cs

@@ -1,9 +1,10 @@
-using System.Runtime.CompilerServices;
+using System;
+using System.Runtime.CompilerServices;
 using Svelto.Common;
 
 namespace Svelto.ECS.DataStructures
 {
-    public struct NativeDynamicArrayCast<T> where T : struct
+    public struct NativeDynamicArrayCast<T>:IDisposable where T : struct
     {
         public NativeDynamicArrayCast(uint size, Allocator allocator)
         {
@@ -61,6 +62,18 @@ namespace Svelto.ECS.DataStructures
         
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public NativeDynamicArray ToNativeArray() { return _array; }
+        
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Set(uint index, in T value)
+        {
+            _array.Set(index, value);
+        }
+        
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Set(int index, in T value)
+        {
+            _array.Set((uint)index, value);
+        }
 
         public bool isValid => _array.isValid;
 

DataStructures/Unmanaged/SharedNativeInt.cs

@@ -1,12 +1,54 @@
 using System;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using Svelto.Common;
 
 namespace Svelto.ECS.DataStructures
 {
+    public struct SharedNative<T> : IDisposable where T : struct, IDisposable
+    {
+#if UNITY_COLLECTIONS || (UNITY_JOBS || UNITY_BURST)
+        [global::Unity.Collections.LowLevel.Unsafe.NativeDisableUnsafePtrRestriction]
+#endif        
+        unsafe IntPtr ptr;
+
+        public SharedNative(in T value)
+        {
+            unsafe
+            {
+                ptr  = MemoryUtilities.Alloc<T>(1, Allocator.Persistent);
+                Unsafe.Write((void*)ptr, value);
+            }
+        }
+
+        public void Dispose()
+        {
+            unsafe
+            {
+                Unsafe.AsRef<T>((void*)ptr).Dispose();
+                
+                MemoryUtilities.Free((IntPtr)ptr, Allocator.Persistent);
+                ptr = IntPtr.Zero;
+            }
+        }
+
+        public ref T value
+        {
+            get
+            {
+                unsafe
+                {
+                    DBC.ECS.Check.Require(ptr != null, "SharedNative has not been initialized");
+
+                    return ref Unsafe.AsRef<T>((void*)ptr);
+                }
+            }
+        }
+    }
+
     public struct SharedNativeInt: IDisposable
     {
-#if UNITY_COLLECTIONS || UNITY_JOBS || UNITY_BURST
+#if UNITY_COLLECTIONS || (UNITY_JOBS || UNITY_BURST)
         [global::Unity.Collections.LowLevel.Unsafe.NativeDisableUnsafePtrRestriction]
 #endif
         unsafe int* data;

DataStructures/Unmanaged/UnsafeBlob.cs

@@ -14,7 +14,7 @@ namespace Svelto.ECS.DataStructures
     ///     Note: this must work inside burst, so it must follow burst restrictions
     ///     It's a typeless native queue based on a ring-buffer model. This means that the writing head and the
     ///     reading head always advance independently. If there is enough space left by dequeued elements,
-    ///     the writing head will wrap around if it reaches the end of the array. The writing head cannot ever surpass the reading head.
+    ///     the writing head will wrap around. The writing head cannot ever surpass the reading head.
     ///  
     /// </summary>
     struct UnsafeBlob : IDisposable
@@ -47,7 +47,7 @@ namespace Svelto.ECS.DataStructures
         internal Allocator allocator;
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal void Enqueue<T>(in T item) where T : struct
+        internal void Enqueue<T>(in T item) where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
@@ -93,7 +93,7 @@ namespace Svelto.ECS.DataStructures
         
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         //The index returned is the index of the unwrapped ring. It must be wrapped again before to be used
-        internal ref T Reserve<T>(out UnsafeArrayIndex index) where T : struct
+        internal ref T Reserve<T>(out UnsafeArrayIndex index) where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
@@ -119,7 +119,7 @@ namespace Svelto.ECS.DataStructures
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal ref T AccessReserved<T>(UnsafeArrayIndex index) where T : struct
+        internal ref T AccessReserved<T>(UnsafeArrayIndex index) where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
@@ -133,7 +133,7 @@ namespace Svelto.ECS.DataStructures
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal T Dequeue<T>() where T : struct
+        internal T Dequeue<T>()  where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
@@ -176,36 +176,39 @@ namespace Svelto.ECS.DataStructures
                 return item;
             }
         }
-
+        
         /// <summary>
-        /// This version of Realloc unwraps a queue, but doesn't change the unwrapped index of existing elements.
-        /// In this way the previously indices will remain valid
+        /// This code unwraps the queue and resizes the array, but doesn't change the unwrapped index of existing elements.
+        /// In this way the previously reserved indices will remain valid
         /// </summary>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal void Realloc(uint newCapacity)
+        internal void Grow<T>() where T : struct //should be unmanaged, but it's not due to Svelto.ECS constraints.
         {
             unsafe
             {
-                //be sure it's multiple of 4. Assuming that what we write is aligned to 4, then we will always have aligned wrapped heads.
+                var sizeOf      = MemoryUtilities.SizeOf<T>();
+
+                var oldCapacity = capacity;
+                
+                uint newCapacity = (uint) ((oldCapacity + sizeOf) << 1);
+                //be sure it's multiple of 4. Assuming that what we write is aligned to 4, then we will always have aligned wrapped heads
                 //the reading and writing head always increment in multiple of 4
                 newCapacity += MemoryUtilities.Pad4(newCapacity);
 
                 byte* newPointer = null;
-#if DEBUG && !PROFILE_SVELTO
-                if (newCapacity <= capacity)
-                    throw new Exception("new capacity must be bigger than current");
-#endif
                 newPointer = (byte*) MemoryUtilities.Alloc(newCapacity, allocator);
 
                 //copy wrapped content if there is any
                 var currentSize = _writeIndex - _readIndex;
                 if (currentSize > 0)
                 {
-                    var oldReaderHead = _readIndex % capacity;
-                    var writerHead = _writeIndex % capacity;
+                    var oldReaderHead = _readIndex % oldCapacity;
+                    var oldWriterHead    = _writeIndex % oldCapacity;
 
-                    //there was no wrapping
-                    if (oldReaderHead < writerHead)
+                    //Remembering that the unwrapped reader cannot ever surpass the unwrapped writer, if the reader is behind the writer
+                    //it means that the writer didn't wrap. It's the natural position so the data can be copied with
+                    //a single memcpy
+                    if (oldReaderHead < oldWriterHead)
                     {
                         var newReaderHead = _readIndex % newCapacity;
                         
@@ -213,15 +216,21 @@ namespace Svelto.ECS.DataStructures
                     }
                     else
                     {
-                        var byteCountToEnd = capacity - oldReaderHead;
-                        var newReaderHead = _readIndex % newCapacity;
+                        //if the wrapped writer is behind the wrapped reader, it means the writer wrapped. Therefore
+                        //I need to copy the data from the current wrapped reader to the end and then from the 
+                        //begin of the array to the current wrapped writer.
+                        
+                        var byteCountToEnd = oldCapacity - oldReaderHead; //bytes to copy from the reader to the end
+                        var newReaderHead  = _readIndex % newCapacity;
                         
 #if DEBUG && !PROFILE_SVELTO
-                        if (newReaderHead + byteCountToEnd + writerHead > newCapacity)
+                        if (newReaderHead + byteCountToEnd + oldWriterHead > newCapacity) //basically the test is the old size must be less than the new capacity. 
                             throw new Exception("something is wrong with my previous assumptions");
 #endif                  
+                        //I am leaving on purpose gap at the begin of the new array if there is any, it will be 
+                        //anyway used once it's time to wrap. 
                         Unsafe.CopyBlock(newPointer + newReaderHead, ptr + oldReaderHead, byteCountToEnd); //from the old reader head to the end of the old array
-                        Unsafe.CopyBlock(newPointer + newReaderHead + byteCountToEnd, ptr + 0, (uint) writerHead); //from the begin of the old array to the old writer head (rember the writerHead wrapped)
+                        Unsafe.CopyBlock(newPointer + newReaderHead + byteCountToEnd, ptr + 0, (uint) oldWriterHead); //from the begin of the old array to the old writer head (rember the writerHead wrapped)
                     }
                 }
 
@@ -231,7 +240,7 @@ namespace Svelto.ECS.DataStructures
                 ptr      = newPointer;
                 capacity = newCapacity;
 
-                //_readIndex  = 0; readIndex won't change to keep the previous reserved indices valid
+                //_readIndex  = 0; the idea is that the old readIndex should remain unchanged. Remember this is the unwrapped index.
                 _writeIndex = _readIndex + currentSize;
             }
         }
@@ -260,4 +269,4 @@ namespace Svelto.ECS.DataStructures
         uint _writeIndex;
         uint _readIndex;
     }
-}
+}

Dispatcher/ReactiveValue.cs

@@ -21,7 +21,7 @@ namespace Svelto.ECS
             _subscriber = callback;
 
             if (notifyImmediately)
-                _subscriber(_senderID, initialValue);
+                _subscriber(senderID, initialValue);
 
             _senderID       = senderID;
             _value          = initialValue;

Extensions/Unity/DOTS/Native.meta → Extensions/Native.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 7ec1b42035f83c698c4d67112a8336c5
+guid: 15fd37f581ab3d8a85d4ed558c555928
 folderAsset: yes
 DefaultImporter:
   externalObjects: {}

Extensions/Unity/DOTS/Native/EnginesRoot.NativeOperation.cs → Extensions/Native/EnginesRoot.NativeOperation.cs

@@ -16,6 +16,7 @@ namespace Svelto.ECS
             //DBC.ECS.Check.Require(EntityDescriptorTemplate<T>.descriptor.isUnmanaged(), "can't remove entities with not native types");
             //todo: remove operation array and store entity descriptor hash in the return value
             //todo I maybe able to provide a  _nativeSwap.SwapEntity<entityDescriptor>
+            //todo make this work with base descriptors too
             _nativeRemoveOperations.Add(new NativeOperationRemove(
                                             EntityDescriptorTemplate<T>.descriptor.componentsToBuild, TypeCache<T>.type
                                           , memberName));
@@ -25,7 +26,7 @@ namespace Svelto.ECS
 
         NativeEntitySwap ProvideNativeEntitySwapQueue<T>(string memberName) where T : IEntityDescriptor, new()
         {
-           // DBC.ECS.Check.Require(EntityDescriptorTemplate<T>.descriptor.isUnmanaged(), "can't swap entities with not native types");
+            // DBC.ECS.Check.Require(EntityDescriptorTemplate<T>.descriptor.isUnmanaged(), "can't swap entities with not native types");
             //todo: remove operation array and store entity descriptor hash in the return value
             _nativeSwapOperations.Add(new NativeOperationSwap(EntityDescriptorTemplate<T>.descriptor.componentsToBuild
                                                             , TypeCache<T>.type, memberName));
@@ -33,19 +34,21 @@ namespace Svelto.ECS
             return new NativeEntitySwap(_nativeSwapOperationQueue, _nativeSwapOperations.count - 1);
         }
 
-        NativeEntityFactory ProvideNativeEntityFactoryQueue<T>(string memberName) where T : IEntityDescriptor, new()
+        NativeEntityFactory ProvideNativeEntityFactoryQueue<T>(string caller) where T : IEntityDescriptor, new()
         {
-            DBC.ECS.Check.Require(EntityDescriptorTemplate<T>.descriptor.IsUnmanaged(), "can't build entities with not native types");
+            DBC.ECS.Check.Require(EntityDescriptorTemplate<T>.descriptor.IsUnmanaged()
+                                , "can't build entities with not native types");
+            DBC.ECS.Check.Require(string.IsNullOrEmpty(caller) == false, "an invalid caller has been provided");
             //todo: remove operation array and store entity descriptor hash in the return value
-            _nativeAddOperations.Add(
-                new NativeOperationBuild(EntityDescriptorTemplate<T>.descriptor.componentsToBuild, TypeCache<T>.type, memberName));
+            _nativeAddOperations.Add(new NativeOperationBuild(EntityDescriptorTemplate<T>.descriptor.componentsToBuild
+                                                            , TypeCache<T>.type, caller));
 
             return new NativeEntityFactory(_nativeAddOperationQueue, _nativeAddOperations.count - 1, _entityLocator);
         }
 
         void FlushNativeOperations(in PlatformProfiler profiler)
         {
-            using (profiler.Sample("Native Remove/Swap Operations"))
+            using (profiler.Sample("Native Remove Operations"))
             {
                 var removeBuffersCount = _nativeRemoveOperationQueue.count;
                 //todo, I don't like that this scans all the queues even if they are empty
@@ -55,17 +58,23 @@ namespace Svelto.ECS
 
                     while (buffer.IsEmpty() == false)
                     {
-                        var                   componentsIndex       = buffer.Dequeue<uint>();
-                        var                   entityEGID            = buffer.Dequeue<EGID>();
-                        NativeOperationRemove nativeRemoveOperation = _nativeRemoveOperations[componentsIndex];
+                        var componentsIndex = buffer.Dequeue<uint>();
+                        var entityEGID      = buffer.Dequeue<EGID>();
+
+                        ref NativeOperationRemove nativeRemoveOperation = ref _nativeRemoveOperations[componentsIndex];
+
                         CheckRemoveEntityID(entityEGID, nativeRemoveOperation.entityDescriptorType
                                           , nativeRemoveOperation.caller);
-                        QueueEntitySubmitOperation(new EntitySubmitOperation(
-                                                       EntitySubmitOperationType.Remove, entityEGID, entityEGID
-                                                     , nativeRemoveOperation.components));
+
+                        QueueRemoveEntityOperation(
+                            entityEGID, FindRealComponents(entityEGID, nativeRemoveOperation.components)
+                          , nativeRemoveOperation.caller);
                     }
                 }
+            }
 
+            using (profiler.Sample("Native Swap Operations"))
+            {
                 var swapBuffersCount = _nativeSwapOperationQueue.count;
                 for (int i = 0; i < swapBuffersCount; i++)
                 {
@@ -76,28 +85,28 @@ namespace Svelto.ECS
                         var componentsIndex = buffer.Dequeue<uint>();
                         var entityEGID      = buffer.Dequeue<DoubleEGID>();
 
-                        var componentBuilders = _nativeSwapOperations[componentsIndex].components;
+                        ref var nativeSwapOperation = ref _nativeSwapOperations[componentsIndex];
 
-                        CheckRemoveEntityID(entityEGID.@from
-                                          , _nativeSwapOperations[componentsIndex].entityDescriptorType
-                                          , _nativeSwapOperations[componentsIndex].caller);
-                        CheckAddEntityID(entityEGID.to, _nativeSwapOperations[componentsIndex].entityDescriptorType
-                                       , _nativeSwapOperations[componentsIndex].caller);
+                        CheckRemoveEntityID(entityEGID.@from, nativeSwapOperation.entityDescriptorType
+                                          , nativeSwapOperation.caller);
+                        CheckAddEntityID(entityEGID.to, nativeSwapOperation.entityDescriptorType
+                                       , nativeSwapOperation.caller);
 
-                        QueueEntitySubmitOperation(new EntitySubmitOperation(
-                                                       EntitySubmitOperationType.Swap, entityEGID.@from, entityEGID.to
-                                                     , componentBuilders));
+                        QueueSwapEntityOperation(entityEGID.@from, entityEGID.to
+                                               , FindRealComponents(entityEGID.@from, nativeSwapOperation.components)
+                                               , nativeSwapOperation.caller);
                     }
                 }
             }
 
+            //todo: it feels weird that these builds in the transient entities database while it could build directly to the final one 
             using (profiler.Sample("Native Add Operations"))
             {
                 var addBuffersCount = _nativeAddOperationQueue.count;
                 for (int i = 0; i < addBuffersCount; i++)
                 {
                     ref var buffer = ref _nativeAddOperationQueue.GetBuffer(i);
-
+                    //todo: I don't like to iterate a constant number of buffer and skip the empty ones
                     while (buffer.IsEmpty() == false)
                     {
                         var componentsIndex = buffer.Dequeue<uint>();
@@ -105,21 +114,24 @@ namespace Svelto.ECS
                         var reference       = buffer.Dequeue<EntityReference>();
                         var componentCounts = buffer.Dequeue<uint>();
 
-                        Check.Assert(egid.groupID.isInvalid == false, "invalid group detected, are you using new ExclusiveGroupStruct() instead of new ExclusiveGroup()?");
+                        Check.Assert(egid.groupID.isInvalid == false
+                                   , "invalid group detected, are you using new ExclusiveGroupStruct() instead of new ExclusiveGroup()?");
 
-                        var componentBuilders    = _nativeAddOperations[componentsIndex].components;
+                        ref var nativeOperation = ref _nativeAddOperations[componentsIndex];
 #if DEBUG && !PROFILE_SVELTO
-                        var entityDescriptorType = _nativeAddOperations[componentsIndex].entityDescriptorType;
-                        CheckAddEntityID(egid, entityDescriptorType, _nativeAddOperations[componentsIndex].caller);
+                        var entityDescriptorType = nativeOperation.entityDescriptorType;
+                        CheckAddEntityID(egid, entityDescriptorType, nativeOperation.caller);
 #endif
 
                         _entityLocator.SetReference(reference, egid);
-                        var dic = EntityFactory.BuildGroupedEntities(egid, _groupedEntityToAdd, componentBuilders
-                                                                   , null
+                        //todo: I reckon is not necessary to carry the components array in the native operation, it's enough to know the descriptor type
+                        //however I guess this can work only if the type is hashed, which could be done with the burst type hash
+                        var dic = EntityFactory.BuildGroupedEntities(egid, _groupedEntityToAdd
+                                                                   , nativeOperation.components, null
 #if DEBUG && !PROFILE_SVELTO
                                                                    , entityDescriptorType
 #endif
-                                                                     );
+                        );
 
                         var init = new EntityInitializer(egid, dic, reference);
 
@@ -149,7 +161,7 @@ namespace Svelto.ECS
         FasterList<NativeOperationRemove> _nativeRemoveOperations;
         FasterList<NativeOperationSwap>   _nativeSwapOperations;
         FasterList<NativeOperationBuild>  _nativeAddOperations;
-        
+
         //todo: I very likely don't need to create one for each native entity factory, the same can be reused
         readonly AtomicNativeBags _nativeAddOperationQueue;
         readonly AtomicNativeBags _nativeRemoveOperationQueue;

Extensions/Unity/DOTS/Native/EnginesRoot.NativeOperation.cs.meta → Extensions/Native/EnginesRoot.NativeOperation.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 2951c865c51635b39dd47906953760d9
+guid: ad3495b942143f628981d218c86d3843
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Extensions/Svelto/EntityNativeDBExtensions.cs → Extensions/Native/EntityNativeDBExtensions.cs

@@ -1,7 +1,5 @@
 using System.Runtime.CompilerServices;
-using Svelto.Common;
 using Svelto.DataStructures;
-using Svelto.DataStructures.Native;
 using Svelto.ECS.Internal;
 
 //todo: once using native memory for unmanaged struct will be optional, this will need to be moved under the Native namespace
@@ -52,7 +50,6 @@ namespace Svelto.ECS
 
             return false;
         }
-        
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool TryGetEntity<T>(this EntitiesDB entitiesDb, uint entityID, ExclusiveGroupStruct @group, out T value)
             where T : unmanaged, IEntityComponent
@@ -66,14 +63,13 @@ namespace Svelto.ECS
             value = default;
             return false;
         }
-        
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool TryGetEntity<T>(this EntitiesDB entitiesDb, EGID egid, out T value)
             where T : unmanaged, IEntityComponent
         {
             return TryGetEntity<T>(entitiesDb, egid.entityID, egid.groupID, out value);
         }
-
+        
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         static bool QueryEntitiesAndIndexInternal<T>
             (this EntitiesDB entitiesDb, EGID entityGID, out uint index, out NB<T> buffer)

Extensions/Svelto/EntityNativeDBExtensions.cs.meta → Extensions/Native/EntityNativeDBExtensions.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 319a6a0269bf3d1bbca89f52ea1d0aeb
+guid: 49472aef2c9c39958b78ceceab4d5be9
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2