diff --git a/Cargo.toml b/Cargo.toml index c2a6787..ad03286 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [features] -default = ["linked"] +# default = ["linked"] linked = ["openxr/linked", "openxr/static"] [dependencies] diff --git a/examples/demo/manifest.yaml b/examples/demo/manifest.yaml index 0fed2f7..ab098d6 100644 --- a/examples/demo/manifest.yaml +++ b/examples/demo/manifest.yaml @@ -7,8 +7,9 @@ android: - name: "android.hardware.vr.headtracking" required: true - name: "oculus.software.handtracking" - required: false + required: false - name: "com.oculus.experimental.enabled" + required: true uses_permission: - name: "com.oculus.permission.HAND_TRACKING" application: diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index 5f24a61..e619829 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -52,6 +52,7 @@ pub fn initialize_xr_graphics( let mut enabled_extensions = xr::ExtensionSet::default(); enabled_extensions.khr_vulkan_enable2 = true; enabled_extensions.khr_convert_timespec_time = true; + enabled_extensions.other.push("XR_METAX2_detached_controllers".into()); #[cfg(target_os = "android")] { enabled_extensions.khr_android_create_instance = true; diff --git a/src/xr_input/hand.rs b/src/xr_input/hand.rs index c2ba04e..853877d 100644 --- a/src/xr_input/hand.rs +++ b/src/xr_input/hand.rs @@ -18,7 +18,7 @@ use super::{ handtracking::HandTrackingTracker, oculus_touch::OculusController, trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker, OpenXRTrackingRoot}, - Hand, QuatConv, + Hand, QuatConv, hands::HandBone, }; /// add debug renderer for controllers @@ -264,97 +264,6 @@ pub fn spawn_hand_entities(mut commands: Commands) { commands.insert_resource(hand_resource); } -#[derive(Component, Debug, Clone, Copy)] -pub enum HandBone { - Palm, - Wrist, - ThumbMetacarpal, - ThumbProximal, - ThumbDistal, - ThumbTip, - IndexMetacarpal, - IndexProximal, - IndexIntermediate, - IndexDistal, - IndexTip, - MiddleMetacarpal, - MiddleProximal, - MiddleIntermediate, - MiddleDistal, - MiddleTip, - RingMetacarpal, - RingProximal, - RingIntermediate, - RingDistal, - RingTip, - LittleMetacarpal, - LittleProximal, - LittleIntermediate, - LittleDistal, - LittleTip, -} -impl HandBone { - pub const fn get_all_bones() -> [HandBone; 26] { - [ - HandBone::Palm, - HandBone::Wrist, - HandBone::ThumbMetacarpal, - HandBone::ThumbProximal, - HandBone::ThumbDistal, - HandBone::ThumbTip, - HandBone::IndexMetacarpal, - HandBone::IndexProximal, - HandBone::IndexIntermediate, - HandBone::IndexDistal, - HandBone::IndexTip, - HandBone::MiddleMetacarpal, - HandBone::MiddleProximal, - HandBone::MiddleIntermediate, - HandBone::MiddleDistal, - HandBone::MiddleTip, - HandBone::RingMetacarpal, - HandBone::RingProximal, - HandBone::RingIntermediate, - HandBone::RingDistal, - HandBone::RingTip, - HandBone::LittleMetacarpal, - HandBone::LittleProximal, - HandBone::LittleIntermediate, - HandBone::LittleDistal, - HandBone::LittleTip, - ] - } - pub fn get_index_from_bone(&self) -> usize { - match &self { - HandBone::Palm => 0, - HandBone::Wrist => 1, - HandBone::ThumbMetacarpal => 2, - HandBone::ThumbProximal => 3, - HandBone::ThumbDistal => 4, - HandBone::ThumbTip => 5, - HandBone::IndexMetacarpal => 6, - HandBone::IndexProximal => 7, - HandBone::IndexIntermediate => 8, - HandBone::IndexDistal => 9, - HandBone::IndexTip => 10, - HandBone::MiddleMetacarpal => 11, - HandBone::MiddleProximal => 12, - HandBone::MiddleIntermediate => 13, - HandBone::MiddleDistal => 14, - HandBone::MiddleTip => 15, - HandBone::RingMetacarpal => 16, - HandBone::RingProximal => 17, - HandBone::RingIntermediate => 18, - HandBone::RingDistal => 19, - HandBone::RingTip => 20, - HandBone::LittleMetacarpal => 21, - HandBone::LittleProximal => 22, - HandBone::LittleIntermediate => 23, - HandBone::LittleDistal => 24, - HandBone::LittleTip => 25, - } - } -} pub fn update_hand_states( oculus_controller: Res, diff --git a/src/xr_input/hands/emulated.rs b/src/xr_input/hands/emulated.rs new file mode 100644 index 0000000..cb263a3 --- /dev/null +++ b/src/xr_input/hands/emulated.rs @@ -0,0 +1,18 @@ +use bevy::prelude::*; + +use crate::xr_input::Hand; + +use super::HandBone; + +#[derive(Deref, DerefMut, Resource)] +pub struct EmulatedHandPose(pub Box (Vec3, Quat) + Send + Sync>); + +pub struct EmulatedHandsPlugin; + +impl Plugin for EmulatedHandsPlugin { + fn build(&self, app: &mut App) { + app.add_systems(Update, update_hand_skeleton_from_emulated); + } +} + +pub(crate) fn update_hand_skeleton_from_emulated() {} diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs new file mode 100644 index 0000000..8612451 --- /dev/null +++ b/src/xr_input/hands/hand_tracking.rs @@ -0,0 +1,164 @@ +use bevy::prelude::*; +use openxr::{HandTracker, Result, SpaceLocationFlags}; + +use crate::{ + input::XrInput, + resources::{XrFrameState, XrSession}, + xr_input::{ + hand::HandBoneRadius, hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv, + Vec3Conv, + }, +}; + +use super::BoneTrackingStatus; + +#[derive(Resource, PartialEq)] +pub enum DisableHandTracking { + OnlyLeft, + OnlyRight, + Both, +} +pub struct HandTrackingPlugin; + +#[derive(Resource)] +pub struct HandTrackingData { + left_hand: HandTracker, + right_hand: HandTracker, +} + +impl HandTrackingData { + pub fn new(session: &XrSession) -> Result { + let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?; + let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?; + Ok(HandTrackingData { + left_hand: left, + right_hand: right, + }) + } + pub fn get_ref<'a>( + &'a self, + input: &'a XrInput, + frame_state: &'a XrFrameState, + ) -> HandTrackingRef<'a> { + HandTrackingRef { + tracking: self, + input, + frame_state, + } + } +} + +pub struct HandTrackingRef<'a> { + tracking: &'a HandTrackingData, + input: &'a XrInput, + frame_state: &'a XrFrameState, +} +#[derive(Debug)] +pub struct HandJoint { + position: Vec3, + position_valid: bool, + position_tracked: bool, + orientaion: Quat, + orientaion_valid: bool, + orientaion_tracked: bool, + radius: f32, +} + +pub struct HandJoints { + inner: [HandJoint; 26], +} + +impl HandJoints { + pub fn get_joint(&self, bone: HandBone) -> &HandJoint { + &self.inner[bone.get_index_from_bone()] + } +} + +impl<'a> HandTrackingRef<'a> { + pub fn get_poses(&self, side: Hand) -> Option { + self.input + .stage + .locate_hand_joints( + match side { + Hand::Left => &self.tracking.left_hand, + Hand::Right => &self.tracking.right_hand, + }, + self.frame_state.lock().unwrap().predicted_display_time, + ) + .unwrap() + .map(|joints| { + joints + .into_iter() + .map(|joint| HandJoint { + position: joint.pose.position.to_vec3(), + orientaion: joint.pose.orientation.to_quat(), + position_valid: joint + .location_flags + .contains(SpaceLocationFlags::POSITION_VALID), + position_tracked: joint + .location_flags + .contains(SpaceLocationFlags::POSITION_TRACKED), + orientaion_valid: joint + .location_flags + .contains(SpaceLocationFlags::ORIENTATION_VALID), + orientaion_tracked: joint + .location_flags + .contains(SpaceLocationFlags::ORIENTATION_TRACKED), + radius: joint.radius, + }) + .collect::>() + .try_into() + .unwrap() + }) + .map(|joints| HandJoints { inner: joints }) + } +} + +impl Plugin for HandTrackingPlugin { + fn build(&self, app: &mut App) { + app.add_systems( + Update, + (update_hand_bones).run_if(|dh: Option>| { + !dh.is_some_and(|v| *v == DisableHandTracking::Both) + }), + ); + } +} + +pub fn update_hand_bones( + hand_tracking: Res, + xr_input: Res, + xr_frame_state: Res, + root_query: Query<(&Transform, With, Without)>, + mut bones: Query<( + &mut Transform, + &Hand, + &HandBone, + &mut HandBoneRadius, + &mut BoneTrackingStatus, + )>, +) { + let hand_ref = hand_tracking.get_ref(&xr_input, &xr_frame_state); + let (root_transform, _, _) = root_query.get_single().unwrap(); + let left_hand_data = hand_ref.get_poses(Hand::Left); + let right_hand_data = hand_ref.get_poses(Hand::Right); + bones + .par_iter_mut() + .for_each(|(mut transform, hand, bone, mut radius, mut status)| { + let bone_data = match (hand, &left_hand_data, &right_hand_data) { + (Hand::Left, Some(data), _) => data.get_joint(*bone), + (Hand::Right, _, Some(data)) => data.get_joint(*bone), + _ => { + *status = BoneTrackingStatus::Emulated; + return; + } + }; + if *status == BoneTrackingStatus::Emulated { + *status = BoneTrackingStatus::Tracked; + } + radius.0 = bone_data.radius; + *transform = transform + .with_translation(root_transform.transform_point(bone_data.position)) + .with_rotation(root_transform.rotation * bone_data.orientaion) + }); +} diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs new file mode 100644 index 0000000..0780adb --- /dev/null +++ b/src/xr_input/hands/mod.rs @@ -0,0 +1,108 @@ +use bevy::prelude::*; + +pub mod emulated; +pub mod hand_tracking; + +pub struct HandsPlugin; + +impl Plugin for HandsPlugin { + fn build(&self, app: &mut bevy::prelude::App) {} +} + +#[derive(Component, Debug, Clone, Copy, PartialEq)] +pub enum BoneTrackingStatus { + Emulated, + Tracked, +} + +#[derive(Component, Debug, Clone, Copy)] +pub enum HandBone { + Palm, + Wrist, + ThumbMetacarpal, + ThumbProximal, + ThumbDistal, + ThumbTip, + IndexMetacarpal, + IndexProximal, + IndexIntermediate, + IndexDistal, + IndexTip, + MiddleMetacarpal, + MiddleProximal, + MiddleIntermediate, + MiddleDistal, + MiddleTip, + RingMetacarpal, + RingProximal, + RingIntermediate, + RingDistal, + RingTip, + LittleMetacarpal, + LittleProximal, + LittleIntermediate, + LittleDistal, + LittleTip, +} +impl HandBone { + pub const fn get_all_bones() -> [HandBone; 26] { + [ + HandBone::Palm, + HandBone::Wrist, + HandBone::ThumbMetacarpal, + HandBone::ThumbProximal, + HandBone::ThumbDistal, + HandBone::ThumbTip, + HandBone::IndexMetacarpal, + HandBone::IndexProximal, + HandBone::IndexIntermediate, + HandBone::IndexDistal, + HandBone::IndexTip, + HandBone::MiddleMetacarpal, + HandBone::MiddleProximal, + HandBone::MiddleIntermediate, + HandBone::MiddleDistal, + HandBone::MiddleTip, + HandBone::RingMetacarpal, + HandBone::RingProximal, + HandBone::RingIntermediate, + HandBone::RingDistal, + HandBone::RingTip, + HandBone::LittleMetacarpal, + HandBone::LittleProximal, + HandBone::LittleIntermediate, + HandBone::LittleDistal, + HandBone::LittleTip, + ] + } + pub fn get_index_from_bone(&self) -> usize { + match &self { + HandBone::Palm => 0, + HandBone::Wrist => 1, + HandBone::ThumbMetacarpal => 2, + HandBone::ThumbProximal => 3, + HandBone::ThumbDistal => 4, + HandBone::ThumbTip => 5, + HandBone::IndexMetacarpal => 6, + HandBone::IndexProximal => 7, + HandBone::IndexIntermediate => 8, + HandBone::IndexDistal => 9, + HandBone::IndexTip => 10, + HandBone::MiddleMetacarpal => 11, + HandBone::MiddleProximal => 12, + HandBone::MiddleIntermediate => 13, + HandBone::MiddleDistal => 14, + HandBone::MiddleTip => 15, + HandBone::RingMetacarpal => 16, + HandBone::RingProximal => 17, + HandBone::RingIntermediate => 18, + HandBone::RingDistal => 19, + HandBone::RingTip => 20, + HandBone::LittleMetacarpal => 21, + HandBone::LittleProximal => 22, + HandBone::LittleIntermediate => 23, + HandBone::LittleDistal => 24, + HandBone::LittleTip => 25, + } + } +} diff --git a/src/xr_input/handtracking.rs b/src/xr_input/handtracking.rs index 1e52d4c..fb8fb89 100644 --- a/src/xr_input/handtracking.rs +++ b/src/xr_input/handtracking.rs @@ -1,15 +1,11 @@ -use std::mem::MaybeUninit; - use bevy::prelude::*; use openxr::{HandJointLocationEXT, HandTracker, Result}; use crate::{ input::XrInput, - resources::{XrFrameState, XrFrameWaiter, XrSession}, + resources::{XrFrameState, XrSession}, }; -use super::hand::HandBone; - #[derive(Resource)] pub struct HandTrackingTracker { left_hand: HandTracker, @@ -44,10 +40,8 @@ pub struct HandTrackingRef<'a> { frame_state: &'a XrFrameState, } -// pub type HandJoints = [(HandJointLocationEXT, HandBone); 26]; - impl<'a> HandTrackingRef<'a> { - pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT;26]> { + pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> { self.input .stage .locate_hand_joints( @@ -55,16 +49,8 @@ impl<'a> HandTrackingRef<'a> { self.frame_state.lock().unwrap().predicted_display_time, ) .unwrap() - // .map(|joints| { - // joints - // .into_iter() - // .zip(HandBone::get_all_bones().into_iter()) - // .collect::>() - // .try_into() - // .unwrap() - // }) } - pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT;26]> { + pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> { self.input .stage .locate_hand_joints( @@ -72,13 +58,5 @@ impl<'a> HandTrackingRef<'a> { self.frame_state.lock().unwrap().predicted_display_time, ) .unwrap() - // .map(|joints| { - // joints - // .into_iter() - // .zip(HandBone::get_all_bones().into_iter()) - // .collect::>() - // .try_into() - // .unwrap() - // }) } } diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index 8842240..a439707 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -8,6 +8,7 @@ pub mod xr_camera; pub mod hand_poses; pub mod hand; pub mod handtracking; +pub mod hands; use crate::resources::XrSession; use crate::xr_begin_frame;