Implement opt-in change indexes for dense components.

Cargo.toml

@@ -5110,6 +5110,10 @@ inherits = "release"
 lto = "fat"
 panic = "abort"
 
+[profile.rel-with-deb-info]
+inherits = "release"
+debug = 1
+
 [package.metadata.docs.rs]
 # This cfg is needed so that #[doc(fake_variadic)] is correctly propagated for
 # impls for re-exported traits. See https://github.com/rust-lang/cargo/issues/8811

benches/benches/bevy_ecs/change_detection/distributions.rs

@@ -0,0 +1,100 @@
+use std::hint::black_box;
+
+use bevy_ecs::{
+    component::{Component, Mutable},
+    entity::Entity,
+    query::{Changed, QueryState},
+    world::World,
+};
+use criterion::{criterion_group, BatchSize, Criterion};
+use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng};
+
+criterion_group!(benches, distributions);
+
+#[derive(Clone, Copy, Component, Default)]
+#[component(change = "indexed")]
+struct IndexedComponent(u32);
+
+#[derive(Clone, Copy, Component, Default)]
+struct NonIndexedComponent(u32);
+
+trait TestComponent: Component<Mutability = Mutable> + Default {
+    fn mutate(&mut self);
+}
+
+impl TestComponent for IndexedComponent {
+    fn mutate(&mut self) {
+        self.0 += 1;
+    }
+}
+
+impl TestComponent for NonIndexedComponent {
+    fn mutate(&mut self) {
+        self.0 += 1;
+    }
+}
+
+fn distributions(criterion: &mut Criterion) {
+    const ENTITY_COUNT: usize = 100000;
+    let mut group = criterion.benchmark_group("distributions");
+
+    for changed_entity_count in [0, 1, 10, 100, 1000, 10000, 100000] {
+        group.bench_function(
+            format!("indexed_distribution_changed_{changed_entity_count}"),
+            |bencher| {
+                bencher.iter_batched_ref(
+                    || setup_benchmark::<IndexedComponent>(changed_entity_count),
+                    |&mut (ref mut world, ref mut query)| {
+                        run_benchmark::<IndexedComponent>(world, query, changed_entity_count);
+                    },
+                    BatchSize::SmallInput,
+                );
+            },
+        );
+        group.bench_function(
+            format!("non_indexed_distribution_changed_{changed_entity_count}"),
+            |bencher| {
+                bencher.iter_batched_ref(
+                    || setup_benchmark::<NonIndexedComponent>(changed_entity_count),
+                    |&mut (ref mut world, ref mut query)| {
+                        run_benchmark::<NonIndexedComponent>(world, query, changed_entity_count);
+                    },
+                    BatchSize::SmallInput,
+                );
+            },
+        );
+    }
+
+    fn setup_benchmark<C>(changed_entity_count: usize) -> (World, QueryState<Entity, Changed<C>>)
+    where
+        C: TestComponent,
+    {
+        let mut world = World::new();
+        let mut order = vec![];
+        for _ in 0..ENTITY_COUNT {
+            order.push(world.spawn(C::default()).id());
+        }
+        order.shuffle(&mut StdRng::seed_from_u64(12345));
+        world.clear_trackers();
+        for entity in &order[0..changed_entity_count] {
+            world.get_mut::<C>(*entity).unwrap().mutate();
+        }
+        let query = world.query_filtered::<Entity, Changed<C>>();
+        (world, query)
+    }
+
+    fn run_benchmark<C>(
+        world: &mut World,
+        query: &mut QueryState<Entity, Changed<C>>,
+        changed_entity_count: usize,
+    ) where
+        C: TestComponent,
+    {
+        let mut count = 0;
+        for entity in query.iter(world) {
+            black_box(entity);
+            count += 1;
+        }
+        assert_eq!(count, changed_entity_count);
+    }
+}

benches/benches/bevy_ecs/change_detection.rs → benches/benches/bevy_ecs/change_detection/mod.rs

@@ -11,6 +11,8 @@ use chacha20::ChaCha8Rng;
 use criterion::{criterion_group, Criterion};
 use rand::{prelude::SliceRandom, SeedableRng};
 
+pub mod distributions;
+
 criterion_group!(
     benches,
     all_added_detection,
@@ -38,6 +40,9 @@ struct Sparse(f32);
 #[derive(Component, Default)]
 #[component(storage = "Table")]
 struct Data<const X: u16>(f32);
+#[derive(Component, Default)]
+#[component(storage = "Table", change = "indexed")]
+struct TableIndexed(f32);
 
 trait BenchModify {
     fn bench_modify(&mut self) -> f32;
@@ -57,6 +62,13 @@ impl BenchModify for Sparse {
     }
 }
 
+impl BenchModify for TableIndexed {
+    fn bench_modify(&mut self) -> f32 {
+        self.0 += 1f32;
+        black_box(self.0)
+    }
+}
+
 const ENTITIES_TO_BENCH_COUNT: &[u32] = &[5000, 50000];
 
 type BenchGroup<'a> = criterion::BenchmarkGroup<'a, criterion::measurement::WallTime>;
@@ -120,6 +132,7 @@ fn all_added_detection(criterion: &mut Criterion) {
             vec![
                 Box::new(all_added_detection_generic::<Table>),
                 Box::new(all_added_detection_generic::<Sparse>),
+                Box::new(all_added_detection_generic::<TableIndexed>),
             ],
             entity_count,
         );
@@ -168,6 +181,7 @@ fn all_changed_detection(criterion: &mut Criterion) {
             vec![
                 Box::new(all_changed_detection_generic::<Table>),
                 Box::new(all_changed_detection_generic::<Sparse>),
+                Box::new(all_changed_detection_generic::<TableIndexed>),
             ],
             entity_count,
         );
@@ -218,6 +232,7 @@ fn few_changed_detection(criterion: &mut Criterion) {
             vec![
                 Box::new(few_changed_detection_generic::<Table>),
                 Box::new(few_changed_detection_generic::<Sparse>),
+                Box::new(few_changed_detection_generic::<TableIndexed>),
             ],
             entity_count,
         );
@@ -262,6 +277,7 @@ fn none_changed_detection(criterion: &mut Criterion) {
             vec![
                 Box::new(none_changed_detection_generic::<Table>),
                 Box::new(none_changed_detection_generic::<Sparse>),
+                Box::new(none_changed_detection_generic::<TableIndexed>),
             ],
             entity_count,
         );
@@ -374,6 +390,11 @@ fn multiple_archetype_none_changed_detection(criterion: &mut Criterion) {
                 archetype_count,
                 entity_count,
             );
+            multiple_archetype_none_changed_detection_generic::<TableIndexed>(
+                &mut group,
+                archetype_count,
+                entity_count,
+            );
         }
     }
 }

benches/benches/bevy_ecs/main.rs

@@ -21,6 +21,7 @@ mod world;
 criterion_main!(
     bundles::benches,
     change_detection::benches,
+    change_detection::distributions::benches,
     components::benches,
     empty_archetypes::benches,
     entity_cloning::benches,

crates/bevy_asset/src/asset_changed.rs

@@ -290,6 +290,7 @@ unsafe impl<A: AsAssetId> WorldQuery for AssetChanged<A> {
 // SAFETY: read-only access
 unsafe impl<A: AsAssetId> QueryFilter for AssetChanged<A> {
     const IS_ARCHETYPAL: bool = false;
+    const USES_INDEX: bool = false;
 
     #[inline]
     unsafe fn filter_fetch(

crates/bevy_camera/src/primitives.rs

@@ -61,6 +61,7 @@ impl MeshAabb for Mesh {
 /// [`Mesh3d`]: bevy_mesh::Mesh
 #[derive(Component, Clone, Copy, Debug, Default, Reflect, PartialEq)]
 #[reflect(Component, Default, Debug, PartialEq, Clone)]
+#[component(change = "indexed")]
 pub struct Aabb {
     pub center: Vec3A,
     pub half_extents: Vec3A,

crates/bevy_camera/src/visibility/mod.rs

@@ -66,6 +66,7 @@ use bevy_mesh::{mark_3d_meshes_as_changed_if_their_assets_changed, Mesh, Mesh2d,
 /// - when overwriting a [`Mesh`]'s transform on the GPU side (e.g. overwriting `MeshInputUniform`'s
 ///   `world_from_local`), resulting in stale CPU-side positions.
 #[derive(Component, Default)]
+#[component(change = "indexed")]
 pub struct NoCpuCulling;
 
 /// User indication of whether an entity is visible. Propagates down the entity hierarchy.
@@ -78,6 +79,7 @@ pub struct NoCpuCulling;
 /// To read the visibility of an entity, query for its [`InheritedVisibility`] instead.
 /// For more information, see [module level documentation](self#what-is-the-difference-between-visibility-components).
 #[derive(Component, Clone, Copy, Reflect, Debug, PartialEq, Eq, Default)]
+#[component(change = "indexed")]
 #[reflect(Component, Default, Debug, PartialEq, Clone)]
 #[require(InheritedVisibility, ViewVisibility)]
 pub enum Visibility {
@@ -161,6 +163,7 @@ impl PartialEq<&Visibility> for Visibility {
 /// [`VisibilityPropagate`]: VisibilitySystems::VisibilityPropagate
 #[derive(Component, Deref, Debug, Default, Clone, Copy, Reflect, PartialEq, Eq)]
 #[reflect(Component, Default, Debug, PartialEq, Clone)]
+#[component(change = "indexed")]
 pub struct InheritedVisibility(bool);
 
 impl InheritedVisibility {
@@ -223,6 +226,7 @@ pub struct VisibilityClass(pub SmallVec<[TypeId; 1]>);
 /// [`CheckVisibility`]: VisibilitySystems::CheckVisibility
 #[derive(Component, Debug, Default, Clone, Copy, Reflect, PartialEq, Eq)]
 #[reflect(Component, Default, Debug, PartialEq, Clone)]
+#[component(change = "indexed")]
 pub struct ViewVisibility(
     /// Bit packed booleans to track current and previous view visibility state.
     ///
@@ -314,6 +318,7 @@ impl<'a> SetViewVisibility for Mut<'a, ViewVisibility> {
 /// - when using some light effects, like wanting a [`Mesh`] out of the [`Frustum`]
 ///   to appear in the reflection of a [`Mesh`] within.
 #[derive(Debug, Component, Default, Reflect, Clone, PartialEq)]
+#[component(change = "indexed")]
 #[reflect(Component, Default, Debug)]
 pub struct NoFrustumCulling;
 
@@ -647,7 +652,7 @@ fn visibility_propagate_system(
     children_query: Query<&Children, (With<Visibility>, With<InheritedVisibility>)>,
     mut removed_child_of: RemovedComponents<ChildOf>,
 ) {
-    for (entity, visibility, child_of, children) in &changed {
+    for (entity, visibility, child_of, children) in changed.iter() {
         let is_visible = match visibility {
             Visibility::Visible => true,
             Visibility::Hidden => false,

crates/bevy_camera/src/visibility/range.rs

@@ -76,6 +76,7 @@ impl Plugin for VisibilityRangePlugin {
 /// `start_margin` of the next lower LOD; this is important for the crossfade
 /// effect to function properly.
 #[derive(Component, Clone, PartialEq, Default, Reflect)]
+#[component(change = "indexed")]
 #[reflect(Component, PartialEq, Hash, Clone)]
 pub struct VisibilityRange {
     /// The range of distances, in world units, between which this entity will

crates/bevy_ecs/macros/src/component.rs

@@ -48,6 +48,7 @@ pub fn derive_resource(input: TokenStream) -> TokenStream {
     });
 
     let storage = storage_path(&bevy_ecs_path, StorageTy::Table);
+    let change_mode = change_mode(&bevy_ecs_path, ChangeMode::Default);
 
     let on_add_path = None;
     let on_remove_path = None;
@@ -87,6 +88,7 @@ pub fn derive_resource(input: TokenStream) -> TokenStream {
     let component_derive_token_stream = TokenStream::from(quote! {
         impl #impl_generics #bevy_ecs_path::component::Component for #struct_name #type_generics #where_clause {
             const STORAGE_TYPE: #bevy_ecs_path::component::StorageType = #storage;
+            const CHANGE_MODE: #bevy_ecs_path::component::ChangeMode = #change_mode;
             type Mutability = #bevy_ecs_path::component::Mutable;
             fn register_required_components(
                 _requiree: #bevy_ecs_path::component::ComponentId,
@@ -245,6 +247,8 @@ pub fn derive_component(input: TokenStream) -> TokenStream {
         .predicates
         .push(parse_quote! { Self: ::core::marker::Send + ::core::marker::Sync + 'static });
 
+    let change_mode = change_mode(&bevy_ecs_path, attrs.change_mode);
+
     let requires = &attrs.requires;
     let mut register_required = Vec::with_capacity(attrs.requires.iter().len());
     if let Some(requires) = requires {
@@ -330,6 +334,7 @@ pub fn derive_component(input: TokenStream) -> TokenStream {
         #required_component_docs
         impl #impl_generics #bevy_ecs_path::component::Component for #struct_name #type_generics #where_clause {
             const STORAGE_TYPE: #bevy_ecs_path::component::StorageType = #storage;
+            const CHANGE_MODE: #bevy_ecs_path::component::ChangeMode = #change_mode;
             type Mutability = #mutable_type;
             fn register_required_components(
                 _requiree: #bevy_ecs_path::component::ComponentId,
@@ -459,6 +464,7 @@ pub const STORAGE: &str = "storage";
 pub const REQUIRE: &str = "require";
 pub const RELATIONSHIP: &str = "relationship";
 pub const RELATIONSHIP_TARGET: &str = "relationship_target";
+pub const CHANGE: &str = "change";
 
 pub const ON_ADD: &str = "on_add";
 pub const ON_INSERT: &str = "on_insert";
@@ -593,6 +599,7 @@ struct Attrs {
     immutable: bool,
     clone_behavior: Option<Expr>,
     map_entities: Option<MapEntitiesAttributeKind>,
+    change_mode: ChangeMode,
 }
 
 #[derive(Clone, Copy)]
@@ -601,6 +608,12 @@ enum StorageTy {
     SparseSet,
 }
 
+#[derive(Clone, Copy)]
+enum ChangeMode {
+    Default,
+    Indexed,
+}
+
 struct Require {
     path: Path,
     func: Option<TokenStream2>,
@@ -620,6 +633,9 @@ struct RelationshipTarget {
 const TABLE: &str = "Table";
 const SPARSE_SET: &str = "SparseSet";
 
+const DEFAULT: &str = "default";
+const INDEXED: &str = "indexed";
+
 fn parse_component_attr(ast: &DeriveInput) -> Result<Attrs> {
     let mut attrs = Attrs {
         storage: StorageTy::Table,
@@ -634,6 +650,7 @@ fn parse_component_attr(ast: &DeriveInput) -> Result<Attrs> {
         immutable: false,
         clone_behavior: None,
         map_entities: None,
+        change_mode: ChangeMode::Default,
     };
 
     let mut require_paths = HashSet::new();
@@ -685,6 +702,17 @@ fn parse_component_attr(ast: &DeriveInput) -> Result<Attrs> {
                 } else if nested.path.is_ident(MAP_ENTITIES) {
                     attrs.map_entities = Some(nested.input.parse::<MapEntitiesAttributeKind>()?);
                     Ok(())
+                } else if nested.path.is_ident(CHANGE) {
+                    attrs.change_mode = match nested.value()?.parse::<LitStr>()?.value() {
+                        s if s == DEFAULT => ChangeMode::Default,
+                        s if s == INDEXED => ChangeMode::Indexed,
+                        s => {
+                            return Err(nested.error(format!(
+                                "Invalid change mode `{s}`, expected '{DEFAULT}' or '{INDEXED}'.",
+                            )));
+                        }
+                    };
+                    Ok(())
                 } else {
                     Err(nested.error("Unsupported attribute"))
                 }
@@ -797,6 +825,15 @@ fn storage_path(bevy_ecs_path: &Path, ty: StorageTy) -> TokenStream2 {
     quote! { #bevy_ecs_path::component::StorageType::#storage_type }
 }
 
+fn change_mode(bevy_ecs_path: &Path, change_mode: ChangeMode) -> TokenStream2 {
+    let change_mode = match change_mode {
+        ChangeMode::Default => Ident::new("Default", Span::call_site()),
+        ChangeMode::Indexed => Ident::new("Indexed", Span::call_site()),
+    };
+
+    quote! { #bevy_ecs_path::component::ChangeMode::#change_mode }
+}
+
 fn hook_register_function_call(
     bevy_ecs_path: &Path,
     hook: TokenStream2,

crates/bevy_ecs/macros/src/lib.rs

@@ -159,7 +159,7 @@ pub fn derive_bundle(input: TokenStream) -> TokenStream {
             #[inline]
             unsafe fn get_components(
                 ptr: #ecs_path::ptr::MovingPtr<'_, Self>,
-                func: &mut impl FnMut(#ecs_path::component::StorageType, #ecs_path::ptr::OwningPtr<'_>)
+                func: &mut impl FnMut(#ecs_path::component::StorageType, #ecs_path::component::ChangeMode, #ecs_path::ptr::OwningPtr<'_>)
             ) {
                 use #ecs_path::__macro_exports::DebugCheckedUnwrap;