Merge branch 'hand_refactor' into demo

Cargo.toml

@@ -30,8 +30,8 @@ openxr = { version = "0.17.1", features = ["static"] }
 [dev-dependencies]
 bevy = "0.12"
 color-eyre = "0.6.2"
-bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
+# bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
-# bevy_rapier3d = { git = "https://github.com/alexichepura/bevy_rapier", version = "0.22.0", branch = "bevy-012"}
+bevy_rapier3d = { git = "https://github.com/alexichepura/bevy_rapier", version = "0.22.0", branch = "bevy-012"}
 
 [workspace]
 members = ["examples/demo"]

examples/demo/manifest.yaml

@@ -9,6 +9,7 @@ android:
       - name: "oculus.software.handtracking"
         required: false
       - name: "com.oculus.experimental.enabled"
+        required: true
     uses_permission:
       - name: "com.oculus.permission.HAND_TRACKING"
     application:

src/graphics/vulkan.rs

@@ -52,6 +52,7 @@ pub fn initialize_xr_graphics(
     let mut enabled_extensions = xr::ExtensionSet::default();
     enabled_extensions.khr_vulkan_enable2 = true;
     enabled_extensions.khr_convert_timespec_time = true;
+    enabled_extensions.other.push("XR_METAX2_detached_controllers".into());
     #[cfg(target_os = "android")]
     {
         enabled_extensions.khr_android_create_instance = true;

src/xr_input/hand.rs

@@ -18,7 +18,7 @@ use super::{
     handtracking::HandTrackingTracker,
     oculus_touch::OculusController,
     trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker, OpenXRTrackingRoot},
-    Hand, QuatConv,
+    Hand, QuatConv, hands::HandBone,
 };
 
 /// add debug renderer for controllers
@@ -264,97 +264,6 @@ pub fn spawn_hand_entities(mut commands: Commands) {
     commands.insert_resource(hand_resource);
 }
 
-#[derive(Component, Debug, Clone, Copy)]
-pub enum HandBone {
-    Palm,
-    Wrist,
-    ThumbMetacarpal,
-    ThumbProximal,
-    ThumbDistal,
-    ThumbTip,
-    IndexMetacarpal,
-    IndexProximal,
-    IndexIntermediate,
-    IndexDistal,
-    IndexTip,
-    MiddleMetacarpal,
-    MiddleProximal,
-    MiddleIntermediate,
-    MiddleDistal,
-    MiddleTip,
-    RingMetacarpal,
-    RingProximal,
-    RingIntermediate,
-    RingDistal,
-    RingTip,
-    LittleMetacarpal,
-    LittleProximal,
-    LittleIntermediate,
-    LittleDistal,
-    LittleTip,
-}
-impl HandBone {
-    pub const fn get_all_bones() -> [HandBone; 26] {
-        [
-            HandBone::Palm,
-            HandBone::Wrist,
-            HandBone::ThumbMetacarpal,
-            HandBone::ThumbProximal,
-            HandBone::ThumbDistal,
-            HandBone::ThumbTip,
-            HandBone::IndexMetacarpal,
-            HandBone::IndexProximal,
-            HandBone::IndexIntermediate,
-            HandBone::IndexDistal,
-            HandBone::IndexTip,
-            HandBone::MiddleMetacarpal,
-            HandBone::MiddleProximal,
-            HandBone::MiddleIntermediate,
-            HandBone::MiddleDistal,
-            HandBone::MiddleTip,
-            HandBone::RingMetacarpal,
-            HandBone::RingProximal,
-            HandBone::RingIntermediate,
-            HandBone::RingDistal,
-            HandBone::RingTip,
-            HandBone::LittleMetacarpal,
-            HandBone::LittleProximal,
-            HandBone::LittleIntermediate,
-            HandBone::LittleDistal,
-            HandBone::LittleTip,
-        ]
-    }
-    pub fn get_index_from_bone(&self) -> usize {
-        match &self {
-            HandBone::Palm => 0,
-            HandBone::Wrist => 1,
-            HandBone::ThumbMetacarpal => 2,
-            HandBone::ThumbProximal => 3,
-            HandBone::ThumbDistal => 4,
-            HandBone::ThumbTip => 5,
-            HandBone::IndexMetacarpal => 6,
-            HandBone::IndexProximal => 7,
-            HandBone::IndexIntermediate => 8,
-            HandBone::IndexDistal => 9,
-            HandBone::IndexTip => 10,
-            HandBone::MiddleMetacarpal => 11,
-            HandBone::MiddleProximal => 12,
-            HandBone::MiddleIntermediate => 13,
-            HandBone::MiddleDistal => 14,
-            HandBone::MiddleTip => 15,
-            HandBone::RingMetacarpal => 16,
-            HandBone::RingProximal => 17,
-            HandBone::RingIntermediate => 18,
-            HandBone::RingDistal => 19,
-            HandBone::RingTip => 20,
-            HandBone::LittleMetacarpal => 21,
-            HandBone::LittleProximal => 22,
-            HandBone::LittleIntermediate => 23,
-            HandBone::LittleDistal => 24,
-            HandBone::LittleTip => 25,
-        }
-    }
-}
 
 pub fn update_hand_states(
     oculus_controller: Res<OculusController>,

src/xr_input/hands/emulated.rs

@@ -0,0 +1,18 @@
+use bevy::prelude::*;
+
+use crate::xr_input::Hand;
+
+use super::HandBone;
+
+#[derive(Deref, DerefMut, Resource)]
+pub struct EmulatedHandPose(pub Box<dyn Fn(Hand, HandBone) -> (Vec3, Quat) + Send + Sync>);
+
+pub struct EmulatedHandsPlugin;
+
+impl Plugin for EmulatedHandsPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(Update, update_hand_skeleton_from_emulated);
+    }
+}
+
+pub(crate) fn update_hand_skeleton_from_emulated() {}

src/xr_input/hands/hand_tracking.rs

@@ -0,0 +1,164 @@
+use bevy::prelude::*;
+use openxr::{HandTracker, Result, SpaceLocationFlags};
+
+use crate::{
+    input::XrInput,
+    resources::{XrFrameState, XrSession},
+    xr_input::{
+        hand::HandBoneRadius, hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv,
+        Vec3Conv,
+    },
+};
+
+use super::BoneTrackingStatus;
+
+#[derive(Resource, PartialEq)]
+pub enum DisableHandTracking {
+    OnlyLeft,
+    OnlyRight,
+    Both,
+}
+pub struct HandTrackingPlugin;
+
+#[derive(Resource)]
+pub struct HandTrackingData {
+    left_hand: HandTracker,
+    right_hand: HandTracker,
+}
+
+impl HandTrackingData {
+    pub fn new(session: &XrSession) -> Result<HandTrackingData> {
+        let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
+        let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
+        Ok(HandTrackingData {
+            left_hand: left,
+            right_hand: right,
+        })
+    }
+    pub fn get_ref<'a>(
+        &'a self,
+        input: &'a XrInput,
+        frame_state: &'a XrFrameState,
+    ) -> HandTrackingRef<'a> {
+        HandTrackingRef {
+            tracking: self,
+            input,
+            frame_state,
+        }
+    }
+}
+
+pub struct HandTrackingRef<'a> {
+    tracking: &'a HandTrackingData,
+    input: &'a XrInput,
+    frame_state: &'a XrFrameState,
+}
+#[derive(Debug)]
+pub struct HandJoint {
+    position: Vec3,
+    position_valid: bool,
+    position_tracked: bool,
+    orientaion: Quat,
+    orientaion_valid: bool,
+    orientaion_tracked: bool,
+    radius: f32,
+}
+
+pub struct HandJoints {
+    inner: [HandJoint; 26],
+}
+
+impl HandJoints {
+    pub fn get_joint(&self, bone: HandBone) -> &HandJoint {
+        &self.inner[bone.get_index_from_bone()]
+    }
+}
+
+impl<'a> HandTrackingRef<'a> {
+    pub fn get_poses(&self, side: Hand) -> Option<HandJoints> {
+        self.input
+            .stage
+            .locate_hand_joints(
+                match side {
+                    Hand::Left => &self.tracking.left_hand,
+                    Hand::Right => &self.tracking.right_hand,
+                },
+                self.frame_state.lock().unwrap().predicted_display_time,
+            )
+            .unwrap()
+            .map(|joints| {
+                joints
+                    .into_iter()
+                    .map(|joint| HandJoint {
+                        position: joint.pose.position.to_vec3(),
+                        orientaion: joint.pose.orientation.to_quat(),
+                        position_valid: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::POSITION_VALID),
+                        position_tracked: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::POSITION_TRACKED),
+                        orientaion_valid: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::ORIENTATION_VALID),
+                        orientaion_tracked: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::ORIENTATION_TRACKED),
+                        radius: joint.radius,
+                    })
+                    .collect::<Vec<HandJoint>>()
+                    .try_into()
+                    .unwrap()
+            })
+            .map(|joints| HandJoints { inner: joints })
+    }
+}
+
+impl Plugin for HandTrackingPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(
+            Update,
+            (update_hand_bones).run_if(|dh: Option<Res<DisableHandTracking>>| {
+                !dh.is_some_and(|v| *v == DisableHandTracking::Both)
+            }),
+        );
+    }
+}
+
+pub fn update_hand_bones(
+    hand_tracking: Res<HandTrackingData>,
+    xr_input: Res<XrInput>,
+    xr_frame_state: Res<XrFrameState>,
+    root_query: Query<(&Transform, With<OpenXRTrackingRoot>, Without<HandBone>)>,
+    mut bones: Query<(
+        &mut Transform,
+        &Hand,
+        &HandBone,
+        &mut HandBoneRadius,
+        &mut BoneTrackingStatus,
+    )>,
+) {
+    let hand_ref = hand_tracking.get_ref(&xr_input, &xr_frame_state);
+    let (root_transform, _, _) = root_query.get_single().unwrap();
+    let left_hand_data = hand_ref.get_poses(Hand::Left);
+    let right_hand_data = hand_ref.get_poses(Hand::Right);
+    bones
+        .par_iter_mut()
+        .for_each(|(mut transform, hand, bone, mut radius, mut status)| {
+            let bone_data = match (hand, &left_hand_data, &right_hand_data) {
+                (Hand::Left, Some(data), _) => data.get_joint(*bone),
+                (Hand::Right, _, Some(data)) => data.get_joint(*bone),
+                _ => {
+                    *status = BoneTrackingStatus::Emulated;
+                    return;
+                }
+            };
+            if *status == BoneTrackingStatus::Emulated {
+                *status = BoneTrackingStatus::Tracked;
+            }
+            radius.0 = bone_data.radius;
+            *transform = transform
+                .with_translation(root_transform.transform_point(bone_data.position))
+                .with_rotation(root_transform.rotation * bone_data.orientaion)
+        });
+}

src/xr_input/hands/mod.rs

@@ -0,0 +1,108 @@
+use bevy::prelude::*;
+
+pub mod emulated;
+pub mod hand_tracking;
+
+pub struct HandsPlugin;
+
+impl Plugin for HandsPlugin {
+    fn build(&self, app: &mut bevy::prelude::App) {}
+}
+
+#[derive(Component, Debug, Clone, Copy, PartialEq)]
+pub enum BoneTrackingStatus {
+    Emulated,
+    Tracked,
+}
+
+#[derive(Component, Debug, Clone, Copy)]
+pub enum HandBone {
+    Palm,
+    Wrist,
+    ThumbMetacarpal,
+    ThumbProximal,
+    ThumbDistal,
+    ThumbTip,
+    IndexMetacarpal,
+    IndexProximal,
+    IndexIntermediate,
+    IndexDistal,
+    IndexTip,
+    MiddleMetacarpal,
+    MiddleProximal,
+    MiddleIntermediate,
+    MiddleDistal,
+    MiddleTip,
+    RingMetacarpal,
+    RingProximal,
+    RingIntermediate,
+    RingDistal,
+    RingTip,
+    LittleMetacarpal,
+    LittleProximal,
+    LittleIntermediate,
+    LittleDistal,
+    LittleTip,
+}
+impl HandBone {
+    pub const fn get_all_bones() -> [HandBone; 26] {
+        [
+            HandBone::Palm,
+            HandBone::Wrist,
+            HandBone::ThumbMetacarpal,
+            HandBone::ThumbProximal,
+            HandBone::ThumbDistal,
+            HandBone::ThumbTip,
+            HandBone::IndexMetacarpal,
+            HandBone::IndexProximal,
+            HandBone::IndexIntermediate,
+            HandBone::IndexDistal,
+            HandBone::IndexTip,
+            HandBone::MiddleMetacarpal,
+            HandBone::MiddleProximal,
+            HandBone::MiddleIntermediate,
+            HandBone::MiddleDistal,
+            HandBone::MiddleTip,
+            HandBone::RingMetacarpal,
+            HandBone::RingProximal,
+            HandBone::RingIntermediate,
+            HandBone::RingDistal,
+            HandBone::RingTip,
+            HandBone::LittleMetacarpal,
+            HandBone::LittleProximal,
+            HandBone::LittleIntermediate,
+            HandBone::LittleDistal,
+            HandBone::LittleTip,
+        ]
+    }
+    pub fn get_index_from_bone(&self) -> usize {
+        match &self {
+            HandBone::Palm => 0,
+            HandBone::Wrist => 1,
+            HandBone::ThumbMetacarpal => 2,
+            HandBone::ThumbProximal => 3,
+            HandBone::ThumbDistal => 4,
+            HandBone::ThumbTip => 5,
+            HandBone::IndexMetacarpal => 6,
+            HandBone::IndexProximal => 7,
+            HandBone::IndexIntermediate => 8,
+            HandBone::IndexDistal => 9,
+            HandBone::IndexTip => 10,
+            HandBone::MiddleMetacarpal => 11,
+            HandBone::MiddleProximal => 12,
+            HandBone::MiddleIntermediate => 13,
+            HandBone::MiddleDistal => 14,
+            HandBone::MiddleTip => 15,
+            HandBone::RingMetacarpal => 16,
+            HandBone::RingProximal => 17,
+            HandBone::RingIntermediate => 18,
+            HandBone::RingDistal => 19,
+            HandBone::RingTip => 20,
+            HandBone::LittleMetacarpal => 21,
+            HandBone::LittleProximal => 22,
+            HandBone::LittleIntermediate => 23,
+            HandBone::LittleDistal => 24,
+            HandBone::LittleTip => 25,
+        }
+    }
+}

src/xr_input/handtracking.rs

@@ -1,15 +1,11 @@
-use std::mem::MaybeUninit;
-
 use bevy::prelude::*;
 use openxr::{HandJointLocationEXT, HandTracker, Result};
 
 use crate::{
     input::XrInput,
-    resources::{XrFrameState, XrFrameWaiter, XrSession},
+    resources::{XrFrameState, XrSession},
 };
 
-use super::hand::HandBone;
-
 #[derive(Resource)]
 pub struct HandTrackingTracker {
     left_hand: HandTracker,
@@ -44,8 +40,6 @@ pub struct HandTrackingRef<'a> {
     frame_state: &'a XrFrameState,
 }
 
-// pub type HandJoints = [(HandJointLocationEXT, HandBone); 26];
-
 impl<'a> HandTrackingRef<'a> {
     pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
         self.input
@@ -55,14 +49,6 @@ impl<'a> HandTrackingRef<'a> {
                 self.frame_state.lock().unwrap().predicted_display_time,
             )
             .unwrap()
-            // .map(|joints| {
-            //     joints
-            //         .into_iter()
-            //         .zip(HandBone::get_all_bones().into_iter())
-            //         .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
-            //         .try_into()
-            //         .unwrap()
-            // })
     }
     pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
         self.input
@@ -72,13 +58,5 @@ impl<'a> HandTrackingRef<'a> {
                 self.frame_state.lock().unwrap().predicted_display_time,
             )
             .unwrap()
-            // .map(|joints| {
-            //     joints
-            //         .into_iter()
-            //         .zip(HandBone::get_all_bones().into_iter())
-            //         .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
-            //         .try_into()
-            //         .unwrap()
-            // })
     }
 }

src/xr_input/mod.rs

@@ -8,6 +8,7 @@ pub mod xr_camera;
 pub mod hand_poses;
 pub mod hand;
 pub mod handtracking;
+pub mod hands;
 
 use crate::resources::XrSession;
 use crate::xr_begin_frame;