Avii/bevy_oxr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

slightly buggy but fully working hand tracking

Browse Source
This commit is contained in:
Schmarni
2023-11-01 21:10:34 +01:00
parent 4c01eee827
commit ccd505051d
8 changed files with 1087 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/xr_input/debug_gizmos.rs
View File

@@ -36,7 +36,12 @@ pub fn draw_gizmos(
xr_input: Res<XrInput>,
instance: Res<XrInstance>,
session: Res<XrSession>,
tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
tracking_root_query: Query<(
&mut Transform,
With<OpenXRTrackingRoot>,
Without<OpenXRLeftController>,
Without<OpenXRRightController>,
)>,
left_controller_query: Query<(
&GlobalTransform,
With<OpenXRLeftController>,
@@ -81,8 +86,7 @@ pub fn draw_gizmos(
Color::LIME_GREEN,
);
}
} else {
info!("right_hand_poses returned None");
return;
}
//lock frame
let frame_state = *frame_state.lock().unwrap();

145
src/xr_input/hand.rs
View File

@@ -1,9 +1,9 @@
use std::f32::consts::PI;
use bevy::prelude::{
default, info, Color, Commands, Component, Entity, Gizmos, GlobalTransform, Plugin, PostUpdate,
PreUpdate, Quat, Query, Res, ResMut, Resource, SpatialBundle, Startup, Transform, Update, Vec3,
With,
default, info, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, GlobalTransform,
Plugin, PostUpdate, PreUpdate, Quat, Query, Res, ResMut, Resource, SpatialBundle, Startup,
Transform, Update, Vec3, With, Without,
};
use openxr::{HandJoint, Posef};
@@ -17,8 +17,8 @@ use super::{
hand_poses::get_simulated_open_hand_transforms,
handtracking::HandTrackingTracker,
oculus_touch::OculusController,
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker},
Hand,
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker, OpenXRTrackingRoot},
Hand, QuatConv,
};
/// add debug renderer for controllers
@@ -53,7 +53,7 @@ pub enum HandInputSource {
impl Default for HandInputSource {
fn default() -> Self {
HandInputSource::Emulated
HandInputSource::OpenXr
}
}
@@ -184,34 +184,7 @@ impl Default for LittleResource {
pub fn spawn_hand_entities(mut commands: Commands) {
let hands = [Hand::Left, Hand::Right];
let bones = [
HandBone::Palm,
HandBone::Wrist,
HandBone::ThumbMetacarpal,
HandBone::ThumbProximal,
HandBone::ThumbDistal,
HandBone::ThumbTip,
HandBone::IndexMetacarpal,
HandBone::IndexProximal,
HandBone::IndexIntermediate,
HandBone::IndexDistal,
HandBone::IndexTip,
HandBone::MiddleMetacarpal,
HandBone::MiddleProximal,
HandBone::MiddleIntermediate,
HandBone::MiddleDistal,
HandBone::MiddleTip,
HandBone::RingMetacarpal,
HandBone::RingProximal,
HandBone::RingIntermediate,
HandBone::RingDistal,
HandBone::RingTip,
HandBone::LittleMetacarpal,
HandBone::LittleProximal,
HandBone::LittleIntermediate,
HandBone::LittleDistal,
HandBone::LittleTip,
];
let bones = HandBone::get_all_bones();
//hand resource
let mut hand_resource = HandsResource { ..default() };
for hand in hands.iter() {
@@ -321,7 +294,7 @@ pub enum HandBone {
LittleTip,
}
impl HandBone {
pub fn get_all_bones() -> [HandBone; 26] {
pub const fn get_all_bones() -> [HandBone; 26] {
[
HandBone::Palm,
HandBone::Wrist,
@@ -351,6 +324,36 @@ impl HandBone {
HandBone::LittleTip,
]
}
pub fn get_index_from_bone(&self) -> usize {
match &self {
HandBone::Palm => 0,
HandBone::Wrist => 1,
HandBone::ThumbMetacarpal => 2,
HandBone::ThumbProximal => 3,
HandBone::ThumbDistal => 4,
HandBone::ThumbTip => 5,
HandBone::IndexMetacarpal => 6,
HandBone::IndexProximal => 7,
HandBone::IndexIntermediate => 8,
HandBone::IndexDistal => 9,
HandBone::IndexTip => 10,
HandBone::MiddleMetacarpal => 11,
HandBone::MiddleProximal => 12,
HandBone::MiddleIntermediate => 13,
HandBone::MiddleDistal => 14,
HandBone::MiddleTip => 15,
HandBone::RingMetacarpal => 16,
HandBone::RingProximal => 17,
HandBone::RingIntermediate => 18,
HandBone::RingDistal => 19,
HandBone::RingTip => 20,
HandBone::LittleMetacarpal => 21,
HandBone::LittleProximal => 22,
HandBone::LittleIntermediate => 23,
HandBone::LittleDistal => 24,
HandBone::LittleTip => 25,
}
}
}
pub fn update_hand_states(
@@ -566,7 +569,15 @@ pub fn update_hand_bones_emulated(
controller_transform: Transform,
hand: Hand,
hand_state: HandState,
hand_bone_query: &mut Query<(&mut Transform, &HandBone, &Hand)>,
hand_bone_query: &mut Query<(
Entity,
&mut Transform,
&HandBone,
&Hand,
Option<&mut HandBoneRadius>,
Without<OpenXRTrackingRoot>,
)>,
) {
let left_hand_rot = Quat::from_rotation_y(180.0 * PI / 180.0);
let hand_translation: Vec3 = match hand {
@@ -854,7 +865,7 @@ pub fn update_hand_bones_emulated(
}
//now that we have all the transforms lets assign them
for (mut transform, handbone, bonehand) in hand_bone_query.iter_mut() {
for (_, mut transform, handbone, bonehand, _, _) in hand_bone_query.iter_mut() {
if *bonehand == hand {
//if the hands match lets go
let index = match_index(handbone);
@@ -1043,12 +1054,23 @@ fn log_hand(hand_pose: [Posef; 26]) {
}
pub fn update_hand_skeletons(
tracking_root_query: Query<(&Transform, With<OpenXRTrackingRoot>)>,
right_controller_query: Query<(&GlobalTransform, With<OpenXRRightController>)>,
left_controller_query: Query<(&GlobalTransform, With<OpenXRLeftController>)>,
hand_states_option: Option<ResMut<HandStatesResource>>,
mut hand_bone_query: Query<(&mut Transform, &HandBone, &Hand)>,
mut commands: Commands,
mut hand_bone_query: Query<(
Entity,
&mut Transform,
&HandBone,
&Hand,
Option<&mut HandBoneRadius>,
Without<OpenXRTrackingRoot>,
)>,
input_source: Option<Res<HandInputSource>>,
hand_tracking: Res<HandTrackingTracker>,
xr_input: Res<XrInput>,
xr_frame_state: Res<XrFrameState>,
) {
match input_source {
Some(res) => match *res {
@@ -1082,7 +1104,35 @@ pub fn update_hand_skeletons(
None => info!("hand states resource not initialized yet"),
}
}
HandInputSource::OpenXr => {}
HandInputSource::OpenXr => {
let hand_ref = hand_tracking.get_ref(&xr_input, &xr_frame_state);
let (root_transform, _) = tracking_root_query.get_single().unwrap();
let left_data = hand_ref.get_left_poses();
let right_data = hand_ref.get_right_poses();
for (entity, mut transform, bone, hand, radius, _) in hand_bone_query.iter_mut() {
let bone_data = match (hand, left_data, right_data) {
(Hand::Left, Some(data), _) => data[bone.get_index_from_bone()],
(Hand::Right, _, Some(data)) => data[bone.get_index_from_bone()],
_ => continue,
};
match radius {
Some(mut r) => r.0 = bone_data.radius,
None => {
commands
.entity(entity)
.insert(HandBoneRadius(bone_data.radius));
}
}
*transform = transform
.with_translation(
root_transform.transform_point(bone_data.pose.position.to_vec3()),
)
.with_rotation(
root_transform.rotation * bone_data.pose.orientation.to_quat(),
)
}
}
},
None => {
info!("hand input source not initialized");
@@ -1091,10 +1141,21 @@ pub fn update_hand_skeletons(
}
}
pub fn draw_hand_entities(mut gizmos: Gizmos, query: Query<(&Transform, &HandBone)>) {
for (transform, hand_bone) in query.iter() {
#[derive(Debug, Component, DerefMut, Deref)]
pub struct HandBoneRadius(pub f32);
pub fn draw_hand_entities(
mut gizmos: Gizmos,
query: Query<(&Transform, &HandBone, Option<&HandBoneRadius>)>,
) {
for (transform, hand_bone, hand_bone_radius) in query.iter() {
let (radius, color) = get_bone_gizmo_style(hand_bone);
gizmos.sphere(transform.translation, transform.rotation, radius, color);
gizmos.sphere(
transform.translation,
transform.rotation,
hand_bone_radius.map_or(radius, |r| r.0),
color,
);
}
}

26
src/xr_input/handtracking.rs
View File

@@ -1,3 +1,5 @@
use std::mem::MaybeUninit;
use bevy::prelude::*;
use openxr::{HandJointLocationEXT, HandTracker, Result};
@@ -6,6 +8,8 @@ use crate::{
resources::{XrFrameState, XrFrameWaiter, XrSession},
};
use super::hand::HandBone;
#[derive(Resource)]
pub struct HandTrackingTracker {
left_hand: HandTracker,
@@ -40,8 +44,10 @@ pub struct HandTrackingRef<'a> {
frame_state: &'a XrFrameState,
}
// pub type HandJoints = [(HandJointLocationEXT, HandBone); 26];
impl<'a> HandTrackingRef<'a> {
pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT;26]> {
self.input
.stage
.locate_hand_joints(
@@ -49,8 +55,16 @@ impl<'a> HandTrackingRef<'a> {
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
// .map(|joints| {
// joints
// .into_iter()
// .zip(HandBone::get_all_bones().into_iter())
// .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
// .try_into()
// .unwrap()
// })
}
pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT;26]> {
self.input
.stage
.locate_hand_joints(
@@ -58,5 +72,13 @@ impl<'a> HandTrackingRef<'a> {
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
// .map(|joints| {
// joints
// .into_iter()
// .zip(HandBone::get_all_bones().into_iter())
// .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
// .try_into()
// .unwrap()
// })
}
}