Basic Handtracking Working

src/xr_input/debug_gizmos.rs

@@ -13,7 +13,11 @@ use crate::xr_input::{
     Hand,
 };
 
-use super::trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot};
+use super::{
+    handtracking::{HandTrackingRef, HandTrackingTracker},
+    trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
+    QuatConv,
+};
 
 /// add debug renderer for controllers
 #[derive(Default)]
@@ -45,7 +49,41 @@ pub fn draw_gizmos(
         Without<OpenXRLeftController>,
         Without<OpenXRTrackingRoot>,
     )>,
+    hand_tracking: Res<HandTrackingTracker>,
 ) {
+    let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
+    if let Some(joints) = handtracking_ref.get_left_poses() {
+        for joint in joints {
+            let p = joint.pose.position;
+            let r = joint.pose.orientation;
+            let quat = r.to_quat();
+            let trans = Transform::from_rotation(quat);
+            gizmos.circle(
+                (p.x, p.y, p.z).into(),
+                trans.forward(),
+                joint.radius,
+                Color::ORANGE_RED,
+            );
+        }
+    } else {
+        info!("left_hand_poses returned None");
+    }
+    if let Some(joints) = handtracking_ref.get_right_poses() {
+        for joint in joints {
+            let p = joint.pose.position;
+            let r = joint.pose.orientation;
+            let quat = r.to_quat();
+            let trans = Transform::from_rotation(quat);
+            gizmos.circle(
+                (p.x, p.y, p.z).into(),
+                trans.forward(),
+                joint.radius,
+                Color::LIME_GREEN,
+            );
+        }
+    } else {
+        info!("right_hand_poses returned None");
+    }
     //lock frame
     let frame_state = *frame_state.lock().unwrap();
     //get controller

src/xr_input/hand.rs

@@ -15,6 +15,7 @@ use crate::{
 
 use super::{
     hand_poses::get_simulated_open_hand_transforms,
+    handtracking::HandTrackingTracker,
     oculus_touch::OculusController,
     trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker},
     Hand,
@@ -319,6 +320,38 @@ pub enum HandBone {
     LittleDistal,
     LittleTip,
 }
+impl HandBone {
+    pub fn get_all_bones() -> [HandBone; 26] {
+        [
+            HandBone::Palm,
+            HandBone::Wrist,
+            HandBone::ThumbMetacarpal,
+            HandBone::ThumbProximal,
+            HandBone::ThumbDistal,
+            HandBone::ThumbTip,
+            HandBone::IndexMetacarpal,
+            HandBone::IndexProximal,
+            HandBone::IndexIntermediate,
+            HandBone::IndexDistal,
+            HandBone::IndexTip,
+            HandBone::MiddleMetacarpal,
+            HandBone::MiddleProximal,
+            HandBone::MiddleIntermediate,
+            HandBone::MiddleDistal,
+            HandBone::MiddleTip,
+            HandBone::RingMetacarpal,
+            HandBone::RingProximal,
+            HandBone::RingIntermediate,
+            HandBone::RingDistal,
+            HandBone::RingTip,
+            HandBone::LittleMetacarpal,
+            HandBone::LittleProximal,
+            HandBone::LittleIntermediate,
+            HandBone::LittleDistal,
+            HandBone::LittleTip,
+        ]
+    }
+}
 
 pub fn update_hand_states(
     oculus_controller: Res<OculusController>,
@@ -1015,6 +1048,7 @@ pub fn update_hand_skeletons(
     hand_states_option: Option<ResMut<HandStatesResource>>,
     mut hand_bone_query: Query<(&mut Transform, &HandBone, &Hand)>,
     input_source: Option<Res<HandInputSource>>,
+    hand_tracking: Res<HandTrackingTracker>,
 ) {
     match input_source {
         Some(res) => match *res {
@@ -1048,10 +1082,7 @@ pub fn update_hand_skeletons(
                     None => info!("hand states resource not initialized yet"),
                 }
             }
-            HandInputSource::OpenXr => {
-                info!("hand input source is open XR: this is not implemented yet");
-                return;
-            }
+            HandInputSource::OpenXr => {}
         },
         None => {
             info!("hand input source not initialized");

src/xr_input/handtracking.rs

@@ -0,0 +1,62 @@
+use bevy::prelude::*;
+use openxr::{HandJointLocationEXT, HandTracker, Result};
+
+use crate::{
+    input::XrInput,
+    resources::{XrFrameState, XrFrameWaiter, XrSession},
+};
+
+#[derive(Resource)]
+pub struct HandTrackingTracker {
+    left_hand: HandTracker,
+    right_hand: HandTracker,
+}
+
+impl HandTrackingTracker {
+    pub fn new(session: &XrSession) -> Result<HandTrackingTracker> {
+        let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
+        let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
+        Ok(HandTrackingTracker {
+            left_hand: left,
+            right_hand: right,
+        })
+    }
+    pub fn get_ref<'a>(
+        &'a self,
+        input: &'a XrInput,
+        frame_state: &'a XrFrameState,
+    ) -> HandTrackingRef<'a> {
+        HandTrackingRef {
+            tracking: self,
+            input,
+            frame_state,
+        }
+    }
+}
+
+pub struct HandTrackingRef<'a> {
+    tracking: &'a HandTrackingTracker,
+    input: &'a XrInput,
+    frame_state: &'a XrFrameState,
+}
+
+impl<'a> HandTrackingRef<'a> {
+    pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
+        self.input
+            .stage
+            .locate_hand_joints(
+                &self.tracking.left_hand,
+                self.frame_state.lock().unwrap().predicted_display_time,
+            )
+            .unwrap()
+    }
+    pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
+        self.input
+            .stage
+            .locate_hand_joints(
+                &self.tracking.right_hand,
+                self.frame_state.lock().unwrap().predicted_display_time,
+            )
+            .unwrap()
+    }
+}

src/xr_input/mod.rs

@@ -7,6 +7,7 @@ pub mod trackers;
 pub mod xr_camera;
 pub mod hand_poses;
 pub mod hand;
+pub mod handtracking;
 
 use crate::resources::XrSession;
 use crate::xr_begin_frame;