Merge pull request #100 from Schmarni-Dev/handtracking

add support for hand tracking and configurable reference spaces

crates/bevy_openxr/Cargo.toml

@@ -8,6 +8,9 @@ default = ["vulkan", "passthrough"]
 vulkan = ["dep:ash"]
 passthrough = []
 
+[dev-dependencies]
+bevy_xr_utils.path = "../bevy_xr_utils"
+
 # bevy can't be placed behind target or proc macros won't work properly
 [dependencies]
 bevy.workspace = true

crates/bevy_openxr/examples/3d_scene.rs

@@ -6,6 +6,7 @@ use bevy_openxr::add_xr_plugins;
 fn main() {
     App::new()
         .add_plugins(add_xr_plugins(DefaultPlugins))
+        .add_plugins(bevy_xr_utils::hand_gizmos::HandGizmosPlugin)
         .add_systems(Startup, setup)
         .run();
 }
@@ -38,5 +39,8 @@ fn setup(
         },
         transform: Transform::from_xyz(4.0, 8.0, 4.0),
         ..default()
+    }); commands.spawn(Camera3dBundle {
+        transform: Transform::from_xyz(-2.5, 4.5, 9.0).looking_at(Vec3::ZERO, Vec3::Y),
+        ..default()
     });
 }

crates/bevy_openxr/examples/android/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2021"
 [dependencies]
 bevy_openxr.path = "../.."
 bevy.workspace = true
+bevy_xr_utils.path = "../../../bevy_xr_utils"
 
 
 [lib]

crates/bevy_openxr/examples/android/src/lib.rs

@@ -1,12 +1,26 @@
 //! A simple 3D scene with light shining over a cube sitting on a plane.
 
 use bevy::prelude::*;
-use bevy_openxr::add_xr_plugins;
+use bevy_openxr::{add_xr_plugins, init::OxrInitPlugin, types::OxrExtensions};
 
 #[bevy_main]
 fn main() {
     App::new()
-        .add_plugins(add_xr_plugins(DefaultPlugins))
+        .add_plugins(add_xr_plugins(DefaultPlugins).set(OxrInitPlugin {
+            app_info: default(),
+            exts: {
+                let mut exts = OxrExtensions::default();
+                exts.enable_fb_passthrough();
+                exts.enable_hand_tracking();
+                exts
+            },
+            blend_modes: default(),
+            backends: default(),
+            formats: default(),
+            resolutions: default(),
+            synchronous_pipeline_compilation: default(),
+        }))
+        .add_plugins(bevy_xr_utils::hand_gizmos::HandGizmosPlugin)
         .add_systems(Startup, setup)
         .insert_resource(ClearColor(Color::NONE))
         .run();

crates/bevy_openxr/src/openxr/action_set_attaching.rs

@@ -14,6 +14,7 @@ impl Plugin for OxrActionAttachingPlugin {
 
 fn attach_sets(session: Res<OxrSession>, mut events: EventReader<OxrAttachActionSet>) {
     let sets = events.read().map(|v| &v.0).collect::<Vec<_>>();
+    if sets.is_empty() {return;}
     info!("attaching {} sessions", sets.len());
     match session.attach_action_sets(&sets) {
         Ok(_) => {info!("attached sessions!")}

crates/bevy_openxr/src/openxr/features/handtracking.rs

@@ -0,0 +1,175 @@
+use bevy::prelude::*;
+use bevy_xr::{
+    hands::{HandBone, HandBoneRadius},
+    session::{session_running, status_changed_to, XrStatus},
+};
+use openxr::SpaceLocationFlags;
+
+use crate::{
+    init::OxrTrackingRoot,
+    reference_space::{OxrPrimaryReferenceSpace, OxrReferenceSpace},
+    resources::{OxrSession, OxrTime},
+};
+
+pub struct HandTrackingPlugin {
+    default_hands: bool,
+}
+impl Default for HandTrackingPlugin {
+    fn default() -> Self {
+        Self {
+            default_hands: true,
+        }
+    }
+}
+
+impl Plugin for HandTrackingPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(PreUpdate, locate_hands.run_if(session_running));
+        if self.default_hands {
+            app.add_systems(
+                PreUpdate,
+                clean_up_default_hands.run_if(status_changed_to(XrStatus::Exiting)),
+            );
+            app.add_systems(
+                PostUpdate,
+                spawn_default_hands.run_if(status_changed_to(XrStatus::Ready)),
+            );
+        }
+    }
+}
+
+fn spawn_default_hands(
+    mut cmds: Commands,
+    session: Res<OxrSession>,
+    root: Query<Entity, With<OxrTrackingRoot>>,
+) {
+    info!("spawning hands");
+    let Ok(root) = root.get_single() else {
+        error!("unable to get tracking root, skipping hand creation");
+        return;
+    };
+    let tracker_left = match session.create_hand_tracker(openxr::HandEXT::LEFT) {
+        Ok(t) => t,
+        Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
+            warn!("Handtracking Extension not loaded, Unable to create Handtracker!");
+            return;
+        }
+        Err(err) => {
+            warn!("Error while creating Handtracker: {}", err.to_string());
+            return;
+        }
+    };
+    let tracker_right = match session.create_hand_tracker(openxr::HandEXT::RIGHT) {
+        Ok(t) => t,
+        Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
+            warn!("Handtracking Extension not loaded, Unable to create Handtracker!");
+            return;
+        }
+        Err(err) => {
+            warn!("Error while creating Handtracker: {}", err.to_string());
+            return;
+        }
+    };
+    let mut left_bones = [Entity::PLACEHOLDER; 26];
+    let mut right_bones = [Entity::PLACEHOLDER; 26];
+    for bone in HandBone::get_all_bones() {
+        let bone_left = cmds
+            .spawn((SpatialBundle::default(), bone, HandBoneRadius(0.0)))
+            .id();
+        let bone_right = cmds
+            .spawn((SpatialBundle::default(), bone, HandBoneRadius(0.0)))
+            .id();
+        cmds.entity(root).push_children(&[bone_right]);
+        left_bones[bone as usize] = bone_left;
+        right_bones[bone as usize] = bone_right;
+    }
+    cmds.spawn((
+        DefaultHandTracker,
+        OxrHandTracker(tracker_left),
+        OxrHandBoneEntities(left_bones),
+    ));
+    cmds.spawn((
+        DefaultHandTracker,
+        OxrHandTracker(tracker_right),
+        OxrHandBoneEntities(right_bones),
+    ));
+}
+
+#[derive(Component)]
+struct DefaultHandTracker;
+#[derive(Component)]
+struct DefaultHandBones;
+
+#[allow(clippy::type_complexity)]
+fn clean_up_default_hands(
+    mut cmds: Commands,
+    query: Query<Entity, Or<(With<DefaultHandTracker>, With<DefaultHandBones>)>>,
+) {
+    for e in &query {
+        cmds.entity(e).despawn();
+    }
+}
+
+#[derive(Deref, DerefMut, Component, Clone, Copy)]
+pub struct OxrHandBoneEntities(pub [Entity; 26]);
+
+#[derive(Deref, DerefMut, Component)]
+pub struct OxrHandTracker(pub openxr::HandTracker);
+
+fn locate_hands(
+    default_ref_space: Res<OxrPrimaryReferenceSpace>,
+    time: Res<OxrTime>,
+    tracker_query: Query<(
+        &OxrHandTracker,
+        Option<&OxrReferenceSpace>,
+        &OxrHandBoneEntities,
+    )>,
+    mut bone_query: Query<(&HandBone, &mut HandBoneRadius, &mut Transform)>,
+) {
+    for (tracker, ref_space, hand_entities) in &tracker_query {
+        let ref_space = ref_space.map(|v| &v.0).unwrap_or(&default_ref_space.0);
+        // relate_hand_joints also provides velocities
+        let joints = match ref_space.locate_hand_joints(tracker, **time) {
+            Ok(Some(v)) => v,
+            Ok(None) => continue,
+            Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
+                error!("HandTracking Extension not loaded");
+                continue;
+            }
+            Err(err) => {
+                warn!("Error while locating hand joints: {}", err.to_string());
+                continue;
+            }
+        };
+        let bone_entities = match bone_query.get_many_mut(hand_entities.0) {
+            Ok(v) => v,
+            Err(err) => {
+                warn!("unable to get entities, {}", err);
+                continue;
+            }
+        };
+        for (bone, mut bone_radius, mut transform) in bone_entities {
+            let joint = joints[*bone as usize];
+            **bone_radius = joint.radius;
+
+            if joint
+                .location_flags
+                .contains(SpaceLocationFlags::POSITION_VALID)
+            {
+                transform.translation.x = joint.pose.position.x;
+                transform.translation.y = joint.pose.position.y;
+                transform.translation.z = joint.pose.position.z;
+            }
+
+            if joint
+                .location_flags
+                .contains(SpaceLocationFlags::ORIENTATION_VALID)
+            {
+                transform.rotation.x = joint.pose.orientation.x;
+                transform.rotation.y = joint.pose.orientation.y;
+                transform.rotation.z = joint.pose.orientation.z;
+                transform.rotation.w = joint.pose.orientation.w;
+            }
+        }
+    }
+}

crates/bevy_openxr/src/openxr/features/mod.rs

@@ -1,2 +1,3 @@
 #[cfg(feature = "passthrough")]
 pub mod passthrough;
+pub mod handtracking;

crates/bevy_openxr/src/openxr/features/passthrough.rs

@@ -83,6 +83,9 @@ pub fn create_passthrough(
 
 #[inline]
 pub fn supports_passthrough(instance: &OxrInstance, system: OxrSystemId) -> Result<bool> {
+    if instance.exts().fb_passthrough.is_none() {
+        return Ok(false);
+    }
     unsafe {
         let mut hand = openxr::sys::SystemPassthroughProperties2FB {
             ty: SystemPassthroughProperties2FB::TYPE,

crates/bevy_openxr/src/openxr/helper_traits.rs

@@ -0,0 +1,90 @@
+use bevy::prelude::*;
+
+pub trait ToPosef {
+    fn to_posef(&self) -> openxr::Posef;
+}
+pub trait ToTransform {
+    fn to_transform(&self) -> Transform;
+}
+pub trait ToQuaternionf {
+    fn to_quaternionf(&self) -> openxr::Quaternionf;
+}
+pub trait ToQuat {
+    fn to_quat(&self) -> Quat;
+}
+pub trait ToVector3f {
+    fn to_vector3f(&self) -> openxr::Vector3f;
+}
+pub trait ToVec3 {
+    fn to_vec3(&self) -> Vec3;
+}
+pub trait ToVector2f {
+    fn to_vector2f(&self) -> openxr::Vector2f;
+}
+pub trait ToVec2 {
+    fn to_vec2(&self) -> Vec2;
+}
+impl ToPosef for Transform {
+    fn to_posef(&self) -> openxr::Posef {
+        openxr::Posef {
+            orientation: self.rotation.to_quaternionf(),
+            position: self.translation.to_vector3f(),
+        }
+    }
+}
+impl ToTransform for openxr::Posef {
+    fn to_transform(&self) -> Transform {
+        Transform::from_translation(self.position.to_vec3())
+            .with_rotation(self.orientation.to_quat())
+    }
+}
+
+impl ToQuaternionf for Quat {
+    fn to_quaternionf(&self) -> openxr::Quaternionf {
+        openxr::Quaternionf {
+            x: self.x,
+            y: self.y,
+            z: self.z,
+            w: self.w,
+        }
+    }
+}
+impl ToQuat for openxr::Quaternionf {
+    fn to_quat(&self) -> Quat {
+        Quat::from_xyzw(self.x, self.y, self.z, self.w)
+    }
+}
+impl ToVector3f for Vec3 {
+    fn to_vector3f(&self) -> openxr::Vector3f {
+        openxr::Vector3f {
+            x: self.x,
+            y: self.y,
+            z: self.z,
+        }
+    }
+}
+impl ToVec3 for openxr::Vector3f {
+    fn to_vec3(&self) -> Vec3 {
+        Vec3 {
+            x: self.x,
+            y: self.y,
+            z: self.z,
+        }
+    }
+}
+impl ToVector2f for Vec2 {
+    fn to_vector2f(&self) -> openxr::Vector2f {
+        openxr::Vector2f {
+            x: self.x,
+            y: self.y,
+        }
+    }
+}
+impl ToVec2 for openxr::Vector2f {
+    fn to_vec2(&self) -> Vec2 {
+        Vec2 {
+            x: self.x,
+            y: self.y,
+        }
+    }
+}

crates/bevy_openxr/src/openxr/init.rs

@@ -28,6 +28,7 @@ use bevy_xr::session::XrStatusChanged;
 
 use crate::error::OxrError;
 use crate::graphics::*;
+use crate::reference_space::OxrPrimaryReferenceSpace;
 use crate::resources::*;
 use crate::types::*;
 
@@ -39,6 +40,8 @@ pub fn session_started(started: Option<Res<OxrSessionStarted>>) -> bool {
 pub enum OxrPreUpdateSet {
     PollEvents,
     HandleEvents,
+    UpdateCriticalComponents,
+    UpdateNonCriticalComponents,
 }
 
 pub struct OxrInitPlugin {
@@ -58,6 +61,24 @@ pub struct OxrInitPlugin {
     /// Passed into the render plugin when added to the app.
     pub synchronous_pipeline_compilation: bool,
 }
+impl Default for OxrInitPlugin {
+    fn default() -> Self {
+        Self {
+            app_info: default(),
+            exts: {
+                let mut exts = OxrExtensions::default();
+                exts.enable_fb_passthrough();
+                exts.enable_hand_tracking();
+                exts
+            },
+            blend_modes: default(),
+            backends: default(),
+            formats: Some(vec![wgpu::TextureFormat::Rgba8UnormSrgb]),
+            resolutions: default(),
+            synchronous_pipeline_compilation: default(),
+        }
+    }
+}
 
 #[derive(Component)]
 pub struct OxrTrackingRoot;
@@ -167,7 +188,10 @@ impl Plugin for OxrInitPlugin {
             (
                 OxrPreUpdateSet::PollEvents.before(handle_session),
                 OxrPreUpdateSet::HandleEvents.after(handle_session),
-            ),
+                OxrPreUpdateSet::UpdateCriticalComponents,
+                OxrPreUpdateSet::UpdateNonCriticalComponents,
+            )
+                .chain(),
         );
 
         let session_started = OxrSessionStarted::default();
@@ -293,7 +317,6 @@ fn init_xr_session(
     OxrSwapchain,
     OxrSwapchainImages,
     OxrGraphicsInfo,
-    OxrStage,
 )> {
     let (session, frame_waiter, frame_stream) =
         unsafe { instance.create_session(system_id, graphics_info)? };
@@ -395,12 +418,6 @@ fn init_xr_session(
     }
     .ok_or(OxrError::NoAvailableBackend)?;
 
-    let stage = OxrStage(
-        session
-            .create_reference_space(openxr::ReferenceSpaceType::STAGE, openxr::Posef::IDENTITY)?
-            .into(),
-    );
-
     let graphics_info = OxrGraphicsInfo {
         blend_mode,
         resolution,
@@ -414,7 +431,6 @@ fn init_xr_session(
         swapchain,
         images,
         graphics_info,
-        stage,
     ))
 }
 
@@ -426,7 +442,6 @@ struct OxrRenderResources {
     swapchain: OxrSwapchain,
     images: OxrSwapchainImages,
     graphics_info: OxrGraphicsInfo,
-    stage: OxrStage,
 }
 
 pub fn create_xr_session(
@@ -442,19 +457,17 @@ pub fn create_xr_session(
         **system_id,
         create_info.clone(),
     ) {
-        Ok((session, frame_waiter, frame_stream, swapchain, images, graphics_info, stage)) => {
+        Ok((session, frame_waiter, frame_stream, swapchain, images, graphics_info)) => {
             commands.insert_resource(session.clone());
             commands.insert_resource(frame_waiter);
             commands.insert_resource(images.clone());
             commands.insert_resource(graphics_info.clone());
-            commands.insert_resource(stage.clone());
             commands.insert_resource(OxrRenderResources {
                 session,
                 frame_stream,
                 swapchain,
                 images,
                 graphics_info,
-                stage,
             });
         }
         Err(e) => error!("Failed to initialize XrSession: {e}"),
@@ -483,7 +496,6 @@ pub fn transfer_xr_resources(mut commands: Commands, mut world: ResMut<MainWorld
         swapchain,
         images,
         graphics_info,
-        stage,
     }) = world.remove_resource()
     else {
         return;
@@ -494,7 +506,6 @@ pub fn transfer_xr_resources(mut commands: Commands, mut world: ResMut<MainWorld
     commands.insert_resource(swapchain);
     commands.insert_resource(images);
     commands.insert_resource(graphics_info);
-    commands.insert_resource(stage);
 }
 
 /// Polls any OpenXR events and handles them accordingly
@@ -547,14 +558,14 @@ pub fn destroy_xr_session(mut commands: Commands) {
     commands.remove_resource::<OxrFrameWaiter>();
     commands.remove_resource::<OxrSwapchainImages>();
     commands.remove_resource::<OxrGraphicsInfo>();
-    commands.remove_resource::<OxrStage>();
+    commands.remove_resource::<OxrPrimaryReferenceSpace>();
     commands.insert_resource(OxrCleanupSession(true));
 }
 
 pub fn destroy_xr_session_render(world: &mut World) {
     world.remove_resource::<OxrSwapchain>();
     world.remove_resource::<OxrFrameStream>();
-    world.remove_resource::<OxrStage>();
+    world.remove_resource::<OxrPrimaryReferenceSpace>();
     world.remove_resource::<OxrSwapchainImages>();
     world.remove_resource::<OxrGraphicsInfo>();
     world.remove_resource::<OxrSession>();

crates/bevy_openxr/src/openxr/layer_builder.rs

@@ -4,6 +4,7 @@ use bevy::ecs::world::World;
 use openxr::{sys, CompositionLayerFlags, Fovf, Posef, Rect2Di, Space};
 
 use crate::graphics::graphics_match;
+use crate::reference_space::OxrPrimaryReferenceSpace;
 use crate::resources::*;
 
 pub trait LayerProvider {
@@ -16,7 +17,7 @@ pub struct PassthroughLayer;
 
 impl LayerProvider for ProjectionLayer {
     fn get<'a>(&self, world: &'a World) -> Box<dyn CompositionLayer<'a> + 'a> {
-        let stage = world.resource::<OxrStage>();
+        let stage = world.resource::<OxrPrimaryReferenceSpace>();
         let openxr_views = world.resource::<OxrViews>();
         let swapchain = world.resource::<OxrSwapchain>();
         let graphics_info = world.resource::<OxrGraphicsInfo>();

crates/bevy_openxr/src/openxr/mod.rs

@@ -10,19 +10,25 @@ use bevy_xr::session::XrSessionPlugin;
 use init::OxrInitPlugin;
 use render::OxrRenderPlugin;
 
-use self::{exts::OxrExtensions, features::passthrough::OxrPassthroughPlugin};
+use self::{
+    exts::OxrExtensions,
+    features::{handtracking::HandTrackingPlugin, passthrough::OxrPassthroughPlugin},
+    reference_space::OxrReferenceSpacePlugin,
+};
 
+pub mod action_binding;
+pub mod action_set_attaching;
 pub mod error;
 mod exts;
 pub mod features;
 pub mod graphics;
+pub mod helper_traits;
 pub mod init;
 pub mod layer_builder;
+pub mod reference_space;
 pub mod render;
 pub mod resources;
 pub mod types;
-pub mod action_binding;
-pub mod action_set_attaching;
 
 pub fn add_xr_plugins<G: PluginGroup>(plugins: G) -> PluginGroupBuilder {
     plugins
@@ -30,37 +36,24 @@ pub fn add_xr_plugins<G: PluginGroup>(plugins: G) -> PluginGroupBuilder {
         .disable::<RenderPlugin>()
         .disable::<PipelinedRenderingPlugin>()
         .add_before::<RenderPlugin, _>(XrSessionPlugin)
-        .add_before::<RenderPlugin, _>(OxrInitPlugin {
+        .add_before::<RenderPlugin, _>(OxrInitPlugin::default())
-            app_info: default(),
+        .add(OxrReferenceSpacePlugin::default())
-            exts: {
-                let mut exts = OxrExtensions::default();
-                exts.enable_fb_passthrough();
-                exts
-            },
-            blend_modes: default(),
-            backends: default(),
-            formats: Some(vec![wgpu::TextureFormat::Rgba8UnormSrgb]),
-            resolutions: default(),
-            synchronous_pipeline_compilation: default(),
-        })
         .add(OxrRenderPlugin)
         .add(OxrPassthroughPlugin)
+        .add(HandTrackingPlugin::default())
         .add(XrCameraPlugin)
         .add(action_set_attaching::OxrActionAttachingPlugin)
         .add(action_binding::OxrActionBindingPlugin)
         // .add(XrActionPlugin)
         .set(WindowPlugin {
-            #[cfg(not(target_os = "android"))]
             primary_window: Some(Window {
                 transparent: true,
                 present_mode: PresentMode::AutoNoVsync,
                 // title: self.app_info.name.clone(),
                 ..default()
             }),
-            #[cfg(target_os = "android")]
+            // #[cfg(target_os = "android")]
-            primary_window: None, // ?
+            // exit_condition: bevy::window::ExitCondition::DontExit,
-            #[cfg(target_os = "android")]
-            exit_condition: bevy::window::ExitCondition::DontExit,
             #[cfg(target_os = "android")]
             close_when_requested: true,
             ..default()

crates/bevy_openxr/src/openxr/reference_space.rs

@@ -0,0 +1,60 @@
+use std::sync::Arc;
+
+use bevy::{
+    prelude::*,
+    render::extract_resource::{ExtractResource, ExtractResourcePlugin},
+};
+use bevy_xr::session::{status_changed_to, XrStatus};
+
+use crate::{init::OxrPreUpdateSet, resources::OxrSession};
+
+pub struct OxrReferenceSpacePlugin {
+    pub default_primary_ref_space: openxr::ReferenceSpaceType,
+}
+impl Default for OxrReferenceSpacePlugin {
+    fn default() -> Self {
+        Self {
+            default_primary_ref_space: openxr::ReferenceSpaceType::STAGE,
+        }
+    }
+}
+
+#[derive(Resource)]
+struct OxrPrimaryReferenceSpaceType(openxr::ReferenceSpaceType);
+// TODO: this will keep the session alive so we need to remove this in the render world too
+/// The Default Reference space used for locating things
+#[derive(Resource, Deref, ExtractResource, Clone)]
+pub struct OxrPrimaryReferenceSpace(pub Arc<openxr::Space>);
+
+/// The Reference space used for locating spaces on this entity
+#[derive(Component)]
+pub struct OxrReferenceSpace(pub openxr::Space);
+
+impl Plugin for OxrReferenceSpacePlugin {
+    fn build(&self, app: &mut App) {
+        app.insert_resource(OxrPrimaryReferenceSpaceType(self.default_primary_ref_space));
+        app.add_plugins(ExtractResourcePlugin::<OxrPrimaryReferenceSpace>::default());
+        app.add_systems(
+            PreUpdate,
+            set_primary_ref_space
+                .run_if(status_changed_to(XrStatus::Ready))
+                .in_set(OxrPreUpdateSet::UpdateCriticalComponents),
+        );
+    }
+}
+
+fn set_primary_ref_space(
+    session: Res<OxrSession>,
+    space_type: Res<OxrPrimaryReferenceSpaceType>,
+    mut cmds: Commands,
+) {
+    match session.create_reference_space(space_type.0, openxr::Posef::IDENTITY) {
+        Ok(space) => {
+            cmds.insert_resource(OxrPrimaryReferenceSpace(Arc::new(space)));
+        }
+        Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
+            error!("Required Extension for Reference Space not loaded");
+        }
+        Err(err) => error!("Error while creating reference space: {}", err.to_string()),
+    };
+}

crates/bevy_openxr/src/openxr/render.rs

@@ -9,10 +9,10 @@ use bevy::{
     },
     transform::TransformSystem,
 };
-use bevy_xr::camera::{XrCamera, XrCameraBundle, XrProjection};
+use bevy_xr::{camera::{XrCamera, XrCameraBundle, XrProjection}, session::session_running};
 use openxr::ViewStateFlags;
 
-use crate::resources::*;
+use crate::{reference_space::OxrPrimaryReferenceSpace, resources::*};
 use crate::{
     init::{session_started, OxrPreUpdateSet},
     layer_builder::ProjectionLayer,
@@ -28,17 +28,17 @@ impl Plugin for OxrRenderPlugin {
                 (
                     init_views.run_if(resource_added::<OxrGraphicsInfo>),
                     wait_frame.run_if(session_started),
-                    locate_views.run_if(session_started),
+                    locate_views.run_if(session_running),
-                    update_views.run_if(session_started),
+                    update_views.run_if(session_running),
                 )
                     .chain()
-                    .after(OxrPreUpdateSet::HandleEvents),
+                    .after(OxrPreUpdateSet::UpdateNonCriticalComponents),
             )
             .add_systems(
                 PostUpdate,
                 (locate_views, update_views)
                     .chain()
-                    .run_if(session_started)
+                    .run_if(session_running)
                     .before(TransformSystem::TransformPropagate),
             );
         app.sub_app_mut(RenderApp)
@@ -47,7 +47,7 @@ impl Plugin for OxrRenderPlugin {
                 (
                     (
                         insert_texture_views,
-                        locate_views,
+                        locate_views.run_if(resource_exists::<OxrPrimaryReferenceSpace>),
                         update_views_render_world,
                     )
                         .chain()
@@ -112,7 +112,7 @@ pub fn wait_frame(mut frame_waiter: ResMut<OxrFrameWaiter>, mut commands: Comman
 
 pub fn locate_views(
     session: Res<OxrSession>,
-    stage: Res<OxrStage>,
+    ref_space: Res<OxrPrimaryReferenceSpace>,
     time: Res<OxrTime>,
     mut openxr_views: ResMut<OxrViews>,
 ) {
@@ -121,7 +121,7 @@ pub fn locate_views(
         .locate_views(
             openxr::ViewConfigurationType::PRIMARY_STEREO,
             **time,
-            &stage,
+            &ref_space,
         )
         .expect("Failed to locate views");
     if openxr_views.len() != xr_views.len() {

crates/bevy_openxr/src/openxr/resources.rs

@@ -377,8 +377,8 @@ impl OxrSwapchain {
 pub struct OxrSwapchainImages(pub Arc<Vec<wgpu::Texture>>);
 
 /// Thread safe wrapper around [openxr::Space] representing the stage.
-#[derive(Deref, Clone, Resource)]
+// #[derive(Deref, Clone, Resource)]
-pub struct OxrStage(pub Arc<openxr::Space>);
+// pub struct OxrStage(pub Arc<openxr::Space>);
 
 /// Stores the latest generated [OxrViews]
 #[derive(Clone, Resource, ExtractResource, Deref, DerefMut)]

crates/bevy_xr/Cargo.toml

@@ -7,4 +7,4 @@ edition = "2021"
 
 [dependencies]
 bevy.workspace = true
-bevy_xr_macros.path = "macros"
+bevy_xr_macros.path = "macros"

crates/bevy_xr/src/hands.rs

@@ -0,0 +1,129 @@
+use bevy::{ecs::component::Component, math::bool, prelude::{Deref, DerefMut}};
+
+#[repr(transparent)]
+#[derive(Clone, Copy, Component, Debug,DerefMut,Deref)]
+pub struct HandBoneRadius(pub f32);
+
+#[repr(u8)]
+#[derive(Clone, Copy, Component, Debug)]
+pub enum HandBone {
+    Palm = 0,
+    Wrist = 1,
+    ThumbMetacarpal = 2,
+    ThumbProximal = 3,
+    ThumbDistal = 4,
+    ThumbTip = 5,
+    IndexMetacarpal = 6,
+    IndexProximal = 7,
+    IndexIntermediate = 8,
+    IndexDistal = 9,
+    IndexTip = 10,
+    MiddleMetacarpal = 11,
+    MiddleProximal = 12,
+    MiddleIntermediate = 13,
+    MiddleDistal = 14,
+    MiddleTip = 15,
+    RingMetacarpal = 16,
+    RingProximal = 17,
+    RingIntermediate = 18,
+    RingDistal = 19,
+    RingTip = 20,
+    LittleMetacarpal = 21,
+    LittleProximal = 22,
+    LittleIntermediate = 23,
+    LittleDistal = 24,
+    LittleTip = 25,
+}
+
+impl HandBone {
+    pub const fn is_metacarpal(&self) -> bool {
+        matches!(
+            self,
+            HandBone::ThumbMetacarpal
+                | HandBone::IndexMetacarpal
+                | HandBone::MiddleMetacarpal
+                | HandBone::RingMetacarpal
+                | HandBone::LittleMetacarpal
+        )
+    }
+    pub const fn is_thumb(&self) -> bool {
+        matches!(
+            self,
+            HandBone::ThumbMetacarpal
+                | HandBone::ThumbProximal
+                | HandBone::ThumbDistal
+                | HandBone::ThumbTip
+        )
+    }
+    pub const fn is_index(&self) -> bool {
+        matches!(
+            self,
+            HandBone::IndexMetacarpal
+                | HandBone::IndexProximal
+                | HandBone::IndexIntermediate
+                | HandBone::IndexDistal
+                | HandBone::IndexTip
+        )
+    }
+    pub const fn is_middle(&self) -> bool {
+        matches!(
+            self,
+            HandBone::MiddleMetacarpal
+                | HandBone::MiddleProximal
+                | HandBone::MiddleIntermediate
+                | HandBone::MiddleDistal
+                | HandBone::MiddleTip
+        )
+    }
+    pub const fn is_ring(&self) -> bool {
+        matches!(
+            self,
+            HandBone::RingMetacarpal
+                | HandBone::RingProximal
+                | HandBone::RingIntermediate
+                | HandBone::RingDistal
+                | HandBone::RingTip
+        )
+    }
+    pub const fn is_little(&self) -> bool {
+        matches!(
+            self,
+            HandBone::LittleMetacarpal
+                | HandBone::LittleProximal
+                | HandBone::LittleIntermediate
+                | HandBone::LittleDistal
+                | HandBone::LittleTip
+        )
+    }
+
+    pub const fn get_all_bones() -> [HandBone; 26] {
+        [
+            HandBone::Palm,
+            HandBone::Wrist,
+            HandBone::ThumbMetacarpal,
+            HandBone::ThumbProximal,
+            HandBone::ThumbDistal,
+            HandBone::ThumbTip,
+            HandBone::IndexMetacarpal,
+            HandBone::IndexProximal,
+            HandBone::IndexIntermediate,
+            HandBone::IndexDistal,
+            HandBone::IndexTip,
+            HandBone::MiddleMetacarpal,
+            HandBone::MiddleProximal,
+            HandBone::MiddleIntermediate,
+            HandBone::MiddleDistal,
+            HandBone::MiddleTip,
+            HandBone::RingMetacarpal,
+            HandBone::RingProximal,
+            HandBone::RingIntermediate,
+            HandBone::RingDistal,
+            HandBone::RingTip,
+            HandBone::LittleMetacarpal,
+            HandBone::LittleProximal,
+            HandBone::LittleIntermediate,
+            HandBone::LittleDistal,
+            HandBone::LittleTip,
+        ]
+    }
+}

crates/bevy_xr/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod actions;
 pub mod camera;
 pub mod session;
-pub mod types;
+pub mod types;
+pub mod hands;

crates/bevy_xr_utils/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "bevy_xr_utils"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bevy.workspace = true
+bevy_xr.path = "../bevy_xr"

crates/bevy_xr_utils/src/hand_gizmos.rs

@@ -0,0 +1,34 @@
+use bevy::{prelude::*, transform::TransformSystem};
+use bevy_xr::hands::{HandBone, HandBoneRadius};
+pub struct HandGizmosPlugin;
+impl Plugin for HandGizmosPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(
+            PostUpdate,
+            draw_hand_gizmos.after(TransformSystem::TransformPropagate),
+        );
+    }
+}
+fn draw_hand_gizmos(
+    mut gizmos: Gizmos,
+    query: Query<(&GlobalTransform, &HandBone, &HandBoneRadius)>,
+) {
+    for (transform, bone, radius) in &query {
+        let pose = transform.compute_transform();
+        gizmos.sphere(pose.translation, pose.rotation, **radius, gizmo_color(bone));
+    }
+}
+
+fn gizmo_color(bone: &HandBone) -> Color {
+    match bone {
+        HandBone::Palm => Color::WHITE,
+        HandBone::Wrist => Color::GRAY,
+        b if b.is_thumb() => Color::RED,
+        b if b.is_index() => Color::ORANGE,
+        b if b.is_middle() => Color::YELLOW,
+        b if b.is_ring() => Color::GREEN,
+        b if b.is_little() => Color::BLUE,
+        // should be impossible to hit
+        _ => Color::rgb(1.0, 0.0, 1.0),
+    }
+}

crates/bevy_xr_utils/src/lib.rs

@@ -0,0 +1,3 @@
+// use bevy::prelude::*;
+pub mod hand_gizmos;
+