From 4c01eee8277dab9811c2c8b3b8074810709e79f7 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Wed, 1 Nov 2023 18:37:32 +0100 Subject: [PATCH] Basic Handtracking Working --- .gitignore | 2 +- Cargo.toml | 5 ++- examples/demo/Cargo.toml | 3 +- examples/demo/manifest.yaml | 8 +++++ src/graphics/vulkan.rs | 4 +++ src/lib.rs | 4 ++- src/xr_input/debug_gizmos.rs | 40 ++++++++++++++++++++++- src/xr_input/hand.rs | 39 ++++++++++++++++++++--- src/xr_input/handtracking.rs | 62 ++++++++++++++++++++++++++++++++++++ src/xr_input/mod.rs | 1 + 10 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 src/xr_input/handtracking.rs diff --git a/.gitignore b/.gitignore index 3cc34a6..77d392a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ **/target **/Cargo.lock -**/runtime_libs +**/runtime_libs/arm64-v8a/* diff --git a/Cargo.toml b/Cargo.toml index 8c6bd82..ad03286 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [features] -default = ["linked"] +# default = ["linked"] linked = ["openxr/linked", "openxr/static"] [dependencies] @@ -23,6 +23,9 @@ color-eyre = "0.6.2" bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" } # bevy_rapier3d = { git = "https://github.com/alexichepura/bevy_rapier", version = "0.22.0", branch = "bevy-012"} +[workspace] +members = ["examples/demo"] + [[example]] name = "xr" path = "examples/xr.rs" diff --git a/examples/demo/Cargo.toml b/examples/demo/Cargo.toml index b44fc2d..69d7776 100644 --- a/examples/demo/Cargo.toml +++ b/examples/demo/Cargo.toml @@ -12,7 +12,8 @@ crate-type = ["cdylib","lib"] bevy = { git = "https://github.com/bevyengine/bevy.git" } # bevy = "0.11.3" # default-features is false because it for some reason complains when trying to statically link openxr -bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"} +bevy_openxr = { path = "../../", default-features = false} +# bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"} bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" } color-eyre = "0.6.2" diff --git a/examples/demo/manifest.yaml b/examples/demo/manifest.yaml index 372be27..0fed2f7 100644 --- a/examples/demo/manifest.yaml +++ b/examples/demo/manifest.yaml @@ -3,6 +3,14 @@ android: - "runtime_libs" manifest: package: "org.bevyengine.demo_openxr_android" + uses_feature: + - name: "android.hardware.vr.headtracking" + required: true + - name: "oculus.software.handtracking" + required: false + - name: "com.oculus.experimental.enabled" + uses_permission: + - name: "com.oculus.permission.HAND_TRACKING" application: label: "Bevy Openxr Android" theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen" diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index 115b583..5f24a61 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -51,10 +51,14 @@ pub fn initialize_xr_graphics( let mut enabled_extensions = xr::ExtensionSet::default(); enabled_extensions.khr_vulkan_enable2 = true; + enabled_extensions.khr_convert_timespec_time = true; #[cfg(target_os = "android")] { enabled_extensions.khr_android_create_instance = true; } + enabled_extensions.ext_hand_tracking = available_extensions.ext_hand_tracking; + // enabled_extensions.ext_hand_joints_motion_range = available_extensions.ext_hand_joints_motion_range; + let available_layers = xr_entry.enumerate_layers()?; info!("available xr layers: {:#?}", available_layers); diff --git a/src/lib.rs b/src/lib.rs index 854675a..69dce84 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ use input::XrInput; use openxr as xr; use resources::*; use xr_input::controllers::XrControllerType; +use xr_input::handtracking::HandTrackingTracker; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -145,7 +146,8 @@ impl Plugin for OpenXrPlugin { .insert_resource(input.clone()) .insert_resource(views.clone()) .insert_resource(frame_state.clone()) - .insert_resource(action_sets.clone()); + .insert_resource(action_sets.clone()) + .insert_resource(HandTrackingTracker::new(&session).unwrap()); let (left, right) = swapchain.get_render_views(); let left = ManualTextureView { diff --git a/src/xr_input/debug_gizmos.rs b/src/xr_input/debug_gizmos.rs index c8659e2..da16855 100644 --- a/src/xr_input/debug_gizmos.rs +++ b/src/xr_input/debug_gizmos.rs @@ -13,7 +13,11 @@ use crate::xr_input::{ Hand, }; -use super::trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot}; +use super::{ + handtracking::{HandTrackingRef, HandTrackingTracker}, + trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot}, + QuatConv, +}; /// add debug renderer for controllers #[derive(Default)] @@ -45,7 +49,41 @@ pub fn draw_gizmos( Without, Without, )>, + hand_tracking: Res, ) { + let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state); + if let Some(joints) = handtracking_ref.get_left_poses() { + for joint in joints { + let p = joint.pose.position; + let r = joint.pose.orientation; + let quat = r.to_quat(); + let trans = Transform::from_rotation(quat); + gizmos.circle( + (p.x, p.y, p.z).into(), + trans.forward(), + joint.radius, + Color::ORANGE_RED, + ); + } + } else { + info!("left_hand_poses returned None"); + } + if let Some(joints) = handtracking_ref.get_right_poses() { + for joint in joints { + let p = joint.pose.position; + let r = joint.pose.orientation; + let quat = r.to_quat(); + let trans = Transform::from_rotation(quat); + gizmos.circle( + (p.x, p.y, p.z).into(), + trans.forward(), + joint.radius, + Color::LIME_GREEN, + ); + } + } else { + info!("right_hand_poses returned None"); + } //lock frame let frame_state = *frame_state.lock().unwrap(); //get controller diff --git a/src/xr_input/hand.rs b/src/xr_input/hand.rs index f23b18f..dc71853 100644 --- a/src/xr_input/hand.rs +++ b/src/xr_input/hand.rs @@ -15,6 +15,7 @@ use crate::{ use super::{ hand_poses::get_simulated_open_hand_transforms, + handtracking::HandTrackingTracker, oculus_touch::OculusController, trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker}, Hand, @@ -319,6 +320,38 @@ pub enum HandBone { LittleDistal, LittleTip, } +impl HandBone { + pub fn get_all_bones() -> [HandBone; 26] { + [ + HandBone::Palm, + HandBone::Wrist, + HandBone::ThumbMetacarpal, + HandBone::ThumbProximal, + HandBone::ThumbDistal, + HandBone::ThumbTip, + HandBone::IndexMetacarpal, + HandBone::IndexProximal, + HandBone::IndexIntermediate, + HandBone::IndexDistal, + HandBone::IndexTip, + HandBone::MiddleMetacarpal, + HandBone::MiddleProximal, + HandBone::MiddleIntermediate, + HandBone::MiddleDistal, + HandBone::MiddleTip, + HandBone::RingMetacarpal, + HandBone::RingProximal, + HandBone::RingIntermediate, + HandBone::RingDistal, + HandBone::RingTip, + HandBone::LittleMetacarpal, + HandBone::LittleProximal, + HandBone::LittleIntermediate, + HandBone::LittleDistal, + HandBone::LittleTip, + ] + } +} pub fn update_hand_states( oculus_controller: Res, @@ -1015,6 +1048,7 @@ pub fn update_hand_skeletons( hand_states_option: Option>, mut hand_bone_query: Query<(&mut Transform, &HandBone, &Hand)>, input_source: Option>, + hand_tracking: Res, ) { match input_source { Some(res) => match *res { @@ -1048,10 +1082,7 @@ pub fn update_hand_skeletons( None => info!("hand states resource not initialized yet"), } } - HandInputSource::OpenXr => { - info!("hand input source is open XR: this is not implemented yet"); - return; - } + HandInputSource::OpenXr => {} }, None => { info!("hand input source not initialized"); diff --git a/src/xr_input/handtracking.rs b/src/xr_input/handtracking.rs new file mode 100644 index 0000000..2521011 --- /dev/null +++ b/src/xr_input/handtracking.rs @@ -0,0 +1,62 @@ +use bevy::prelude::*; +use openxr::{HandJointLocationEXT, HandTracker, Result}; + +use crate::{ + input::XrInput, + resources::{XrFrameState, XrFrameWaiter, XrSession}, +}; + +#[derive(Resource)] +pub struct HandTrackingTracker { + left_hand: HandTracker, + right_hand: HandTracker, +} + +impl HandTrackingTracker { + pub fn new(session: &XrSession) -> Result { + let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?; + let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?; + Ok(HandTrackingTracker { + left_hand: left, + right_hand: right, + }) + } + pub fn get_ref<'a>( + &'a self, + input: &'a XrInput, + frame_state: &'a XrFrameState, + ) -> HandTrackingRef<'a> { + HandTrackingRef { + tracking: self, + input, + frame_state, + } + } +} + +pub struct HandTrackingRef<'a> { + tracking: &'a HandTrackingTracker, + input: &'a XrInput, + frame_state: &'a XrFrameState, +} + +impl<'a> HandTrackingRef<'a> { + pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> { + self.input + .stage + .locate_hand_joints( + &self.tracking.left_hand, + self.frame_state.lock().unwrap().predicted_display_time, + ) + .unwrap() + } + pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> { + self.input + .stage + .locate_hand_joints( + &self.tracking.right_hand, + self.frame_state.lock().unwrap().predicted_display_time, + ) + .unwrap() + } +} diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index 8746a17..8842240 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -7,6 +7,7 @@ pub mod trackers; pub mod xr_camera; pub mod hand_poses; pub mod hand; +pub mod handtracking; use crate::resources::XrSession; use crate::xr_begin_frame;