diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index e530521..314f488 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -29,7 +29,7 @@ use bevy_oxr::{ oculus_touch::OculusController, prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}, trackers::{OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker}, - Hand, + Hand, actions::XrActionSets, }, DefaultXrPlugins, }; diff --git a/src/lib.rs b/src/lib.rs index b24b291..392bb24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,12 +23,10 @@ use openxr as xr; use resources::*; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::EmulatedHandsPlugin; -use xr_input::hands::hand_tracking::HandTrackingPlugin; +use xr_input::hands::hand_tracking::{HandTrackingPlugin, HandTrackingData}; use xr_input::handtracking::HandTrackingTracker; use xr_input::OpenXrInput; -use crate::xr_input::oculus_touch::ActionSets; - const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; pub const LEFT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(1208214591); @@ -91,7 +89,7 @@ impl Plugin for OpenXrPlugin { views, frame_state, hand_tracking_enabled, - ) = graphics::initialize_xr_graphics(primary_window,self.0).unwrap(); + ) = graphics::initialize_xr_graphics(primary_window, self.0).unwrap(); // std::thread::sleep(Duration::from_secs(5)); debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); debug!("Configured wgpu adapter Features: {:#?}", device.features()); @@ -163,10 +161,9 @@ impl Plugin for OpenXrPlugin { .insert_resource(action_sets.clone()); let hands = xr_instance.exts().ext_hand_tracking.is_some(); if hands { - app.insert_resource(HandTrackingTracker::new(&session).unwrap()); - app.insert_resource(HandInputSource::OpenXr); + app.insert_resource(HandTrackingData::new(&session).unwrap()); } else { - app.insert_resource(HandInputSource::Emulated); + app.insert_resource(DisableHandTracking::Both); } let (left, right) = swapchain.get_render_views(); @@ -224,7 +221,8 @@ impl PluginGroup for DefaultXrPlugins { .disable::() .add_before::(OpenXrPlugin::default()) .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) - .add(EmulatedHandsPlugin).add(HandTrackingPlugin) + .add(EmulatedHandsPlugin) + .add(HandTrackingPlugin) .set(WindowPlugin { #[cfg(not(target_os = "android"))] primary_window: Some(Window { diff --git a/src/xr_input/debug_gizmos.rs b/src/xr_input/debug_gizmos.rs index caf8cdd..e15e6a9 100644 --- a/src/xr_input/debug_gizmos.rs +++ b/src/xr_input/debug_gizmos.rs @@ -17,7 +17,7 @@ use super::{ actions::XrActionSets, handtracking::{HandTrackingRef, HandTrackingTracker}, trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot}, - QuatConv, + QuatConv, hands::hand_tracking::HandTrackingData, }; /// add debug renderer for controllers @@ -55,19 +55,16 @@ pub fn draw_gizmos( Without, Without, )>, - hand_tracking: Option>, + hand_tracking: Option>, action_sets: Res, ) { if let Some(hand_tracking) = hand_tracking { let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state); - if let Some(joints) = handtracking_ref.get_left_poses() { - for joint in joints { - let p = joint.pose.position; - let r = joint.pose.orientation; - let quat = r.to_quat(); - let trans = Transform::from_rotation(quat); + if let Some(joints) = handtracking_ref.get_poses(Hand::Left) { + for joint in joints.inner() { + let trans = Transform::from_rotation(joint.orientation); gizmos.circle( - (p.x, p.y, p.z).into(), + joint.position, trans.forward(), joint.radius, Color::ORANGE_RED, @@ -76,14 +73,11 @@ pub fn draw_gizmos( } else { info!("left_hand_poses returned None"); } - if let Some(joints) = handtracking_ref.get_right_poses() { - for joint in joints { - let p = joint.pose.position; - let r = joint.pose.orientation; - let quat = r.to_quat(); - let trans = Transform::from_rotation(quat); + if let Some(joints) = handtracking_ref.get_poses(Hand::Right) { + for joint in joints.inner() { + let trans = Transform::from_rotation(joint.orientation); gizmos.circle( - (p.x, p.y, p.z).into(), + joint.position, trans.forward(), joint.radius, Color::LIME_GREEN, diff --git a/src/xr_input/hand.rs b/src/xr_input/hand.rs index c57131d..e168752 100644 --- a/src/xr_input/hand.rs +++ b/src/xr_input/hand.rs @@ -16,10 +16,11 @@ use crate::{ use super::{ actions::XrActionSets, hand_poses::get_simulated_open_hand_transforms, + hands::{BoneTrackingStatus, HandBone}, handtracking::HandTrackingTracker, oculus_touch::OculusController, trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker, OpenXRTrackingRoot}, - Hand, QuatConv, hands::{HandBone, BoneTrackingStatus}, + Hand, QuatConv, }; /// add debug renderer for controllers @@ -32,8 +33,8 @@ impl Plugin for OpenXrHandInput { // .add_systems(Update, update_hand_skeletons) // .add_systems(PreUpdate, update_hand_states) .add_systems(Startup, spawn_hand_entities); - // .insert_resource(HandStatesResource::default()) - // .insert_resource(HandInputSource::default()); + // .insert_resource(HandStatesResource::default()) + // .insert_resource(HandInputSource::default()); } } @@ -198,6 +199,7 @@ pub fn spawn_hand_entities(mut commands: Commands) { OpenXRTracker, hand.clone(), BoneTrackingStatus::Emulated, + HandBoneRadius(1.0), )) .id(); match hand { @@ -267,7 +269,6 @@ pub fn spawn_hand_entities(mut commands: Commands) { commands.insert_resource(hand_resource); } - pub fn update_hand_states( oculus_controller: Res, hand_states_option: Option>, @@ -1063,20 +1064,20 @@ pub struct HandBoneRadius(pub f32); pub fn draw_hand_entities( mut gizmos: Gizmos, - query: Query<(&Transform, &HandBone, Option<&HandBoneRadius>)>, + query: Query<(&Transform, &HandBone, &HandBoneRadius)>, ) { for (transform, hand_bone, hand_bone_radius) in query.iter() { - let (radius, color) = get_bone_gizmo_style(hand_bone); + let (_, color) = get_bone_gizmo_style(hand_bone); gizmos.sphere( transform.translation, transform.rotation, - hand_bone_radius.map_or(radius, |r| r.0), + hand_bone_radius.0, color, ); } } -fn get_bone_gizmo_style(hand_bone: &HandBone) -> (f32, Color) { +pub(crate) fn get_bone_gizmo_style(hand_bone: &HandBone) -> (f32, Color) { match hand_bone { HandBone::Palm => (0.01, Color::WHITE), HandBone::Wrist => (0.01, Color::GRAY), diff --git a/src/xr_input/hands/emulated.rs b/src/xr_input/hands/emulated.rs index b16e230..e1cb915 100644 --- a/src/xr_input/hands/emulated.rs +++ b/src/xr_input/hands/emulated.rs @@ -6,15 +6,19 @@ use openxr::{Action, ActionTy, Binding, HandJoint}; use crate::{ resources::{XrInstance, XrSession}, xr_input::{ + actions::{ + ActionHandednes, ActionType, SetupActionSet, SetupActionSets, XrActionSets, XrBinding, + }, controllers::Touchable, + hand::{get_bone_gizmo_style, HandBoneRadius}, hand_poses::get_simulated_open_hand_transforms, oculus_touch::ActionSets, - trackers::{OpenXRLeftController, OpenXRRightController}, + trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot}, Hand, InteractionProfileBindings, }, }; -use super::HandBone; +use super::{BoneTrackingStatus, HandBone}; pub enum TouchValue { None, @@ -32,152 +36,119 @@ pub struct EmulatedHandsPlugin; impl Plugin for EmulatedHandsPlugin { fn build(&self, app: &mut App) { app.add_systems(PreUpdate, update_hand_skeleton_from_emulated); - app.add_systems( - Startup, - setup_hand_emulation_action_set.map(|res| res.unwrap()), - ); + app.add_systems(Startup, setup_hand_emulation_action_set); } } -#[derive(Resource)] -pub struct HandEmulationActionSet { - thumb_touch: Action, - thumb_x: Action, - thumb_y: Action, - index: Touchable, - middle: Touchable, - ring: Touchable, - little: Touchable, -} -fn setup_hand_emulation_action_set( - instance: Res, - session: Res, - mut action_sets: ResMut, - mut commands: Commands, - mut bindings: ResMut -) -> anyhow::Result<()> { - let left_path = instance.string_to_path("/user/hand/left").unwrap(); - let right_path = instance.string_to_path("/user/hand/right").unwrap(); - let hands = [left_path, right_path]; - // This unwrap Should not trigger since both strings are not empty - let action_set = instance - .create_action_set("hand_pose_approximation_set", "Hand Pose Approximaiton", 0) - .unwrap(); - let hand_action_set = HandEmulationActionSet { - thumb_touch: action_set.create_action::("thumb_touch", "Thumb Touched", &hands)?, - thumb_x: action_set.create_action::("thumb_x", "Thumb X", &hands)?, - thumb_y: action_set.create_action::("thumb_y", "Thumb Y", &hands)?, +const HAND_ACTION_SET: &'static str = "hand_pose_approx"; - index: Touchable:: { - inner: action_set.create_action("index_value", "Index Finger Pull", &hands)?, - touch: action_set.create_action("index_touch", "Index Finger Touch", &hands)?, - }, - middle: Touchable:: { - inner: action_set.create_action("middle_value", "Middle Finger Pull", &hands)?, - touch: action_set.create_action("middle_touch", "Middle Finger Touch", &hands)?, - }, - ring: Touchable:: { - inner: action_set.create_action("ring_value", "Ring Finger Pull", &hands)?, - touch: action_set.create_action("ring_touch", "Ring Finger Touch", &hands)?, - }, - little: Touchable:: { - inner: action_set.create_action("little_value", "Little Finger Pull", &hands)?, - touch: action_set.create_action("little_touch", "Little Finger Touch", &hands)?, - }, - }; +fn setup_hand_emulation_action_set(mut action_sets: ResMut) { + let mut action_set = action_sets.add_action_set(HAND_ACTION_SET, "Hand Pose Approximaiton", 0); + action_set.new_action( + "thumb_touch", + "Thumb Touched", + ActionType::Bool, + ActionHandednes::Double, + ); + action_set.new_action( + "thumb_x", + "Thumb X", + ActionType::F32, + ActionHandednes::Double, + ); + action_set.new_action( + "thumb_y", + "Thumb Y", + ActionType::F32, + ActionHandednes::Double, + ); - suggest_oculus_touch_profile(&instance, &hand_action_set,bindings)?; + action_set.new_action( + "index_touch", + "Index Finger Touched", + ActionType::Bool, + ActionHandednes::Double, + ); + action_set.new_action( + "index_value", + "Index Finger Pull", + ActionType::F32, + ActionHandednes::Double, + ); - session.attach_action_sets(&[&action_set])?; + action_set.new_action( + "middle_value", + "Middle Finger Pull", + ActionType::F32, + ActionHandednes::Double, + ); + action_set.new_action( + "ring_value", + "Ring Finger Pull", + ActionType::F32, + ActionHandednes::Double, + ); + action_set.new_action( + "little_value", + "Little Finger Pull", + ActionType::F32, + ActionHandednes::Double, + ); - action_sets.0.push(action_set); - - commands.insert_resource(hand_action_set); - - Ok(()) + suggest_oculus_touch_profile(&mut action_set); } pub struct EmulatedHandPoseData {} -fn bind<'a, T: ActionTy>( - action: &'a Action, - path: &str, - i: &XrInstance, - bindings: &mut Vec>, -) -> anyhow::Result<()> { - bindings.push(Binding::new( - &action, - i.string_to_path(&("/user/hand/left/input".to_string() + path))?, - )); - bindings.push(Binding::new( - &action, - i.string_to_path(&("/user/hand/right/input".to_string() + path))?, - )); - Ok(()) -} -fn bind_single<'a, T: ActionTy>( - action: &'a Action, - path: &str, - hand: Hand, - i: &XrInstance, - bindings: &mut Vec>, -) -> anyhow::Result<()> { - match hand { - Hand::Left => bindings.push(Binding::new( - &action, - i.string_to_path(&("/user/hand/left/input".to_string() + path))?, - )), - Hand::Right => bindings.push(Binding::new( - &action, - i.string_to_path(&("/user/hand/right/input".to_string() + path))?, - )), - } - Ok(()) -} - -fn suggest_oculus_touch_profile( - i: &XrInstance, - action_set: &HandEmulationActionSet, - mut bindings: ResMut -) -> anyhow::Result<()> { - let mut b = bindings.entry("/interaction_profiles/oculus/touch_controller").or_default(); - bind(&action_set.thumb_x, "/thumbstick/x", i, &mut b)?; - bind(&action_set.thumb_y, "/thumbstick/y", i, &mut b)?; - bind(&action_set.thumb_touch, "/thumbstick/touch", i, &mut b)?; - bind(&action_set.thumb_touch, "/thumbrest/touch", i, &mut b)?; - // bind_single(&action_set.thumb_touch, "/x/touch", Hand::Left, i, &mut b)?; - // bind_single(&action_set.thumb_touch, "/y/touch", Hand::Left, i, &mut b)?; - // bind_single(&action_set.thumb_touch, "/a/touch", Hand::Right, i, &mut b)?; - // bind_single(&action_set.thumb_touch, "/b/touch", Hand::Right, i, &mut b)?; - - // bind(&action_set.index.touch, "/trigger/touch", i, &mut b)?; - // bind(&action_set.index.inner, "/trigger/value", i, &mut b)?; - // - // bind(&action_set.middle.touch, "/squeeze/touch", i, &mut b)?; - // bind(&action_set.middle.inner, "/squeeze/value", i, &mut b)?; - // bind(&action_set.ring.touch, "/squeeze/touch", i, &mut b)?; - // bind(&action_set.ring.inner, "/squeeze/value", i, &mut b)?; - // bind(&action_set.little.touch, "/squeeze/touch", i, &mut b)?; - // bind(&action_set.little.inner, "/squeeze/value", i, &mut b)?; - - i.suggest_interaction_profile_bindings( - i.string_to_path("/interaction_profiles/oculus/touch_controller")?, - &b, - )?; - Ok(()) +fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) { + action_set.suggest_binding( + "/interaction_profiles/oculus/touch_controller", + &[ + XrBinding::new("thumb_x", "/user/hand/left/input/thumbstick/x"), + XrBinding::new("thumb_x", "/user/hand/right/input/thumbstick/x"), + XrBinding::new("thumb_y", "/user/hand/left/input/thumbstick/y"), + XrBinding::new("thumb_y", "/user/hand/right/input/thumbstick/y"), + XrBinding::new("thumb_touch", "/user/hand/left/input/thumbstick/touch"), + XrBinding::new("thumb_touch", "/user/hand/right/input/thumbstick/touch"), + XrBinding::new("thumb_touch", "/user/hand/left/input/x/touch"), + XrBinding::new("thumb_touch", "/user/hand/left/input/y/touch"), + XrBinding::new("thumb_touch", "/user/hand/right/input/a/touch"), + XrBinding::new("thumb_touch", "/user/hand/right/input/b/touch"), + XrBinding::new("thumb_touch", "/user/hand/left/input/thumbrest/touch"), + XrBinding::new("thumb_touch", "/user/hand/right/input/thumbrest/touch"), + XrBinding::new("index_touch", "/user/hand/left/input/trigger/touch"), + XrBinding::new("index_value", "/user/hand/left/input/trigger/value"), + XrBinding::new("index_touch", "/user/hand/right/input/trigger/touch"), + XrBinding::new("index_value", "/user/hand/right/input/trigger/value"), + XrBinding::new("middle_value", "/user/hand/left/input/squeeze/value"), + XrBinding::new("middle_value", "/user/hand/right/input/squeeze/value"), + XrBinding::new("ring_value", "/user/hand/left/input/squeeze/value"), + XrBinding::new("ring_value", "/user/hand/right/input/squeeze/value"), + XrBinding::new("little_value", "/user/hand/left/input/squeeze/value"), + XrBinding::new("little_value", "/user/hand/right/input/squeeze/value"), + ], + ); } pub(crate) fn update_hand_skeleton_from_emulated( session: Res, instance: Res, - action_set: Res, + action_sets: Res, left_controller_transform: Query<&Transform, With>, right_controller_transform: Query<&Transform, With>, + tracking_root_transform: Query<&Transform, With>, mut bones: Query< - (&mut Transform, &HandBone, &Hand), + ( + &mut Transform, + &HandBone, + &Hand, + &BoneTrackingStatus, + &mut HandBoneRadius, + ), ( Without, Without, + Without, ), >, ) { @@ -192,8 +163,9 @@ pub(crate) fn update_hand_skeleton_from_emulated( Hand::Right, ), ] { - let thumb_curl = match action_set - .thumb_touch + let thumb_curl = match action_sets + .get_action_bool(HAND_ACTION_SET, "thumb_touch") + .unwrap() .state(&session, subaction_path) .unwrap() .current_state @@ -201,27 +173,27 @@ pub(crate) fn update_hand_skeleton_from_emulated( true => 1.0, false => 0.0, }; - let index_curl = action_set - .index - .inner + let index_curl = action_sets + .get_action_f32(HAND_ACTION_SET, "index_value") + .unwrap() .state(&session, subaction_path) .unwrap() .current_state; - let middle_curl = action_set - .middle - .inner + let middle_curl = action_sets + .get_action_f32(HAND_ACTION_SET, "middle_value") + .unwrap() .state(&session, subaction_path) .unwrap() .current_state; - let ring_curl = action_set - .ring - .inner + let ring_curl = action_sets + .get_action_f32(HAND_ACTION_SET, "ring_value") + .unwrap() .state(&session, subaction_path) .unwrap() .current_state; - let little_curl = action_set - .little - .inner + let little_curl = action_sets + .get_action_f32(HAND_ACTION_SET, "little_value") + .unwrap() .state(&session, subaction_path) .unwrap() .current_state; @@ -241,11 +213,21 @@ pub(crate) fn update_hand_skeleton_from_emulated( little_curl, ); } - for (mut t, bone, hand) in bones.iter_mut() { + let trt = tracking_root_transform.single(); + for (mut t, bone, hand, status, mut radius) in bones.iter_mut() { + match status { + BoneTrackingStatus::Emulated => {} + BoneTrackingStatus::Tracked => continue, + } + radius.0 = get_bone_gizmo_style(bone).0; + *t = data[match hand { Hand::Left => 0, Hand::Right => 1, - }][bone.get_index_from_bone()] + }][bone.get_index_from_bone()]; + *t = t.with_scale(trt.scale); + *t = t.with_rotation(trt.rotation * t.rotation); + *t = t.with_translation(trt.transform_point(t.translation)); } } pub fn update_hand_bones_emulated( @@ -537,6 +519,15 @@ pub fn update_hand_bones_emulated( calc_transforms } -fn get_bone_curl_angle(bone: HandJoint, thumb_curl: f32) -> f32 { - todo!() +fn get_bone_curl_angle(bone: HandJoint, curl: f32) -> f32 { + let mul: f32 = match bone { + HandJoint::INDEX_PROXIMAL => 0.0, + HandJoint::MIDDLE_PROXIMAL => 0.0, + HandJoint::RING_PROXIMAL => 0.0, + HandJoint::LITTLE_PROXIMAL => 0.0, + HandJoint::THUMB_PROXIMAL => 0.0, + _ => 1.0, + }; + let curl_angle = -((mul * curl * 80.0) + 5.0); + return curl_angle; } diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index 982c919..d51bcf2 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -55,18 +55,23 @@ pub struct HandTrackingRef<'a> { } #[derive(Debug)] pub struct HandJoint { - position: Vec3, - position_valid: bool, - position_tracked: bool, - orientaion: Quat, - orientaion_valid: bool, - orientaion_tracked: bool, - radius: f32, + pub position: Vec3, + pub position_valid: bool, + pub position_tracked: bool, + pub orientation: Quat, + pub orientation_valid: bool, + pub orientation_tracked: bool, + pub radius: f32, } pub struct HandJoints { inner: [HandJoint; 26], } +impl HandJoints { + pub fn inner(&self) -> &[HandJoint; 26] { + &self.inner + } +} impl HandJoints { pub fn get_joint(&self, bone: HandBone) -> &HandJoint { @@ -91,17 +96,17 @@ impl<'a> HandTrackingRef<'a> { .into_iter() .map(|joint| HandJoint { position: joint.pose.position.to_vec3(), - orientaion: joint.pose.orientation.to_quat(), + orientation: joint.pose.orientation.to_quat(), position_valid: joint .location_flags .contains(SpaceLocationFlags::POSITION_VALID), position_tracked: joint .location_flags .contains(SpaceLocationFlags::POSITION_TRACKED), - orientaion_valid: joint + orientation_valid: joint .location_flags .contains(SpaceLocationFlags::ORIENTATION_VALID), - orientaion_tracked: joint + orientation_tracked: joint .location_flags .contains(SpaceLocationFlags::ORIENTATION_TRACKED), radius: joint.radius, @@ -126,8 +131,8 @@ impl Plugin for HandTrackingPlugin { } pub fn update_hand_bones( - disabled_tracking: Res, - hand_tracking: Res, + disabled_tracking: Option>, + hand_tracking: Option>, xr_input: Res, xr_frame_state: Res, root_query: Query<(&Transform, With, Without)>, @@ -139,19 +144,25 @@ pub fn update_hand_bones( &mut BoneTrackingStatus, )>, ) { - let hand_ref = hand_tracking.get_ref(&xr_input, &xr_frame_state); + let hand_ref = match hand_tracking.as_ref() { + Some(h) => h.get_ref(&xr_input, &xr_frame_state), + None => { + warn!("No Handtracking data!"); + return; + } + }; let (root_transform, _, _) = root_query.get_single().unwrap(); let left_hand_data = hand_ref.get_poses(Hand::Left); let right_hand_data = hand_ref.get_poses(Hand::Right); bones .par_iter_mut() .for_each(|(mut transform, hand, bone, mut radius, mut status)| { - match (&hand, disabled_tracking.as_ref()) { - (Hand::Left, DisableHandTracking::OnlyLeft) => { + match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) { + (Hand::Left, Some(DisableHandTracking::OnlyLeft)) => { *status = BoneTrackingStatus::Emulated; return; } - (Hand::Right, DisableHandTracking::OnlyRight) => { + (Hand::Right, Some(DisableHandTracking::OnlyRight)) => { *status = BoneTrackingStatus::Emulated; return; } @@ -171,6 +182,6 @@ pub fn update_hand_bones( radius.0 = bone_data.radius; *transform = transform .with_translation(root_transform.transform_point(bone_data.position)) - .with_rotation(root_transform.rotation * bone_data.orientaion) + .with_rotation(root_transform.rotation * bone_data.orientation) }); }