Basic Handtracking Working

.gitignore

@@ -1,3 +1,3 @@
 **/target
 **/Cargo.lock
-**/runtime_libs
+**/runtime_libs/arm64-v8a/*

Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [features]
-default = ["linked"]
+# default = ["linked"]
 linked = ["openxr/linked", "openxr/static"]
 
 [dependencies]
@@ -23,6 +23,9 @@ color-eyre = "0.6.2"
 bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
 # bevy_rapier3d = { git = "https://github.com/alexichepura/bevy_rapier", version = "0.22.0", branch = "bevy-012"}
 
+[workspace]
+members = ["examples/demo"]
+
 [[example]]
 name = "xr"
 path = "examples/xr.rs"

examples/demo/Cargo.toml

@@ -12,7 +12,8 @@ crate-type = ["cdylib","lib"]
 bevy = { git = "https://github.com/bevyengine/bevy.git" }
 # bevy = "0.11.3"
 # default-features is false because it for some reason complains when trying to statically link openxr
-bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"}
+bevy_openxr = { path = "../../", default-features = false}
+# bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"}
 bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
 color-eyre = "0.6.2"
 

examples/demo/manifest.yaml

@@ -3,6 +3,14 @@ android:
     - "runtime_libs"
   manifest:
     package: "org.bevyengine.demo_openxr_android"
+    uses_feature:
+      - name: "android.hardware.vr.headtracking"
+        required: true
+      - name: "oculus.software.handtracking"
+        required: false 
+      - name: "com.oculus.experimental.enabled"
+    uses_permission:
+      - name: "com.oculus.permission.HAND_TRACKING"
     application:
       label: "Bevy Openxr Android"
       theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen"

src/graphics/vulkan.rs

@@ -51,10 +51,14 @@ pub fn initialize_xr_graphics(
 
     let mut enabled_extensions = xr::ExtensionSet::default();
     enabled_extensions.khr_vulkan_enable2 = true;
+    enabled_extensions.khr_convert_timespec_time = true;
     #[cfg(target_os = "android")]
     {
         enabled_extensions.khr_android_create_instance = true;
     }
+    enabled_extensions.ext_hand_tracking = available_extensions.ext_hand_tracking;
+    // enabled_extensions.ext_hand_joints_motion_range = available_extensions.ext_hand_joints_motion_range;
+    
 
     let available_layers = xr_entry.enumerate_layers()?;
     info!("available xr layers: {:#?}", available_layers);

src/lib.rs

@@ -21,6 +21,7 @@ use input::XrInput;
 use openxr as xr;
 use resources::*;
 use xr_input::controllers::XrControllerType;
+use xr_input::handtracking::HandTrackingTracker;
 use xr_input::OpenXrInput;
 
 const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
@@ -145,7 +146,8 @@ impl Plugin for OpenXrPlugin {
                 .insert_resource(input.clone())
                 .insert_resource(views.clone())
                 .insert_resource(frame_state.clone())
-                .insert_resource(action_sets.clone());
+                .insert_resource(action_sets.clone())
+                .insert_resource(HandTrackingTracker::new(&session).unwrap());
 
             let (left, right) = swapchain.get_render_views();
             let left = ManualTextureView {

src/xr_input/debug_gizmos.rs

@@ -13,7 +13,11 @@ use crate::xr_input::{
     Hand,
 };
 
-use super::trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot};
+use super::{
+    handtracking::{HandTrackingRef, HandTrackingTracker},
+    trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
+    QuatConv,
+};
 
 /// add debug renderer for controllers
 #[derive(Default)]
@@ -45,7 +49,41 @@ pub fn draw_gizmos(
         Without<OpenXRLeftController>,
         Without<OpenXRTrackingRoot>,
     )>,
+    hand_tracking: Res<HandTrackingTracker>,
 ) {
+    let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
+    if let Some(joints) = handtracking_ref.get_left_poses() {
+        for joint in joints {
+            let p = joint.pose.position;
+            let r = joint.pose.orientation;
+            let quat = r.to_quat();
+            let trans = Transform::from_rotation(quat);
+            gizmos.circle(
+                (p.x, p.y, p.z).into(),
+                trans.forward(),
+                joint.radius,
+                Color::ORANGE_RED,
+            );
+        }
+    } else {
+        info!("left_hand_poses returned None");
+    }
+    if let Some(joints) = handtracking_ref.get_right_poses() {
+        for joint in joints {
+            let p = joint.pose.position;
+            let r = joint.pose.orientation;
+            let quat = r.to_quat();
+            let trans = Transform::from_rotation(quat);
+            gizmos.circle(
+                (p.x, p.y, p.z).into(),
+                trans.forward(),
+                joint.radius,
+                Color::LIME_GREEN,
+            );
+        }
+    } else {
+        info!("right_hand_poses returned None");
+    }
     //lock frame
     let frame_state = *frame_state.lock().unwrap();
     //get controller

src/xr_input/hand.rs

@@ -15,6 +15,7 @@ use crate::{
 
 use super::{
     hand_poses::get_simulated_open_hand_transforms,
+    handtracking::HandTrackingTracker,
     oculus_touch::OculusController,
     trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker},
     Hand,
@@ -319,6 +320,38 @@ pub enum HandBone {
     LittleDistal,
     LittleTip,
 }
+impl HandBone {
+    pub fn get_all_bones() -> [HandBone; 26] {
+        [
+            HandBone::Palm,
+            HandBone::Wrist,
+            HandBone::ThumbMetacarpal,
+            HandBone::ThumbProximal,
+            HandBone::ThumbDistal,
+            HandBone::ThumbTip,
+            HandBone::IndexMetacarpal,
+            HandBone::IndexProximal,
+            HandBone::IndexIntermediate,
+            HandBone::IndexDistal,
+            HandBone::IndexTip,
+            HandBone::MiddleMetacarpal,
+            HandBone::MiddleProximal,
+            HandBone::MiddleIntermediate,
+            HandBone::MiddleDistal,
+            HandBone::MiddleTip,
+            HandBone::RingMetacarpal,
+            HandBone::RingProximal,
+            HandBone::RingIntermediate,
+            HandBone::RingDistal,
+            HandBone::RingTip,
+            HandBone::LittleMetacarpal,
+            HandBone::LittleProximal,
+            HandBone::LittleIntermediate,
+            HandBone::LittleDistal,
+            HandBone::LittleTip,
+        ]
+    }
+}
 
 pub fn update_hand_states(
     oculus_controller: Res<OculusController>,
@@ -1015,6 +1048,7 @@ pub fn update_hand_skeletons(
     hand_states_option: Option<ResMut<HandStatesResource>>,
     mut hand_bone_query: Query<(&mut Transform, &HandBone, &Hand)>,
     input_source: Option<Res<HandInputSource>>,
+    hand_tracking: Res<HandTrackingTracker>,
 ) {
     match input_source {
         Some(res) => match *res {
@@ -1048,10 +1082,7 @@ pub fn update_hand_skeletons(
                     None => info!("hand states resource not initialized yet"),
                 }
             }
-            HandInputSource::OpenXr => {
+            HandInputSource::OpenXr => {}
-                info!("hand input source is open XR: this is not implemented yet");
-                return;
-            }
         },
         None => {
             info!("hand input source not initialized");

src/xr_input/handtracking.rs

@@ -0,0 +1,62 @@
+use bevy::prelude::*;
+use openxr::{HandJointLocationEXT, HandTracker, Result};
+
+use crate::{
+    input::XrInput,
+    resources::{XrFrameState, XrFrameWaiter, XrSession},
+};
+
+#[derive(Resource)]
+pub struct HandTrackingTracker {
+    left_hand: HandTracker,
+    right_hand: HandTracker,
+}
+
+impl HandTrackingTracker {
+    pub fn new(session: &XrSession) -> Result<HandTrackingTracker> {
+        let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
+        let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
+        Ok(HandTrackingTracker {
+            left_hand: left,
+            right_hand: right,
+        })
+    }
+    pub fn get_ref<'a>(
+        &'a self,
+        input: &'a XrInput,
+        frame_state: &'a XrFrameState,
+    ) -> HandTrackingRef<'a> {
+        HandTrackingRef {
+            tracking: self,
+            input,
+            frame_state,
+        }
+    }
+}
+
+pub struct HandTrackingRef<'a> {
+    tracking: &'a HandTrackingTracker,
+    input: &'a XrInput,
+    frame_state: &'a XrFrameState,
+}
+
+impl<'a> HandTrackingRef<'a> {
+    pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
+        self.input
+            .stage
+            .locate_hand_joints(
+                &self.tracking.left_hand,
+                self.frame_state.lock().unwrap().predicted_display_time,
+            )
+            .unwrap()
+    }
+    pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
+        self.input
+            .stage
+            .locate_hand_joints(
+                &self.tracking.right_hand,
+                self.frame_state.lock().unwrap().predicted_display_time,
+            )
+            .unwrap()
+    }
+}

src/xr_input/mod.rs

@@ -7,6 +7,7 @@ pub mod trackers;
 pub mod xr_camera;
 pub mod hand_poses;
 pub mod hand;
+pub mod handtracking;
 
 use crate::resources::XrSession;
 use crate::xr_begin_frame;