Avii/bevy_oxr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

started hand (and probably slight openxr extension) refactor. :3

Browse Source
This commit is contained in:
Schmarni
2023-11-04 17:26:30 +01:00
parent ccd505051d
commit 538000899c
9 changed files with 299 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/xr_input/hand.rs
View File

@@ -18,7 +18,7 @@ use super::{
handtracking::HandTrackingTracker,
oculus_touch::OculusController,
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker, OpenXRTrackingRoot},
Hand, QuatConv,
Hand, QuatConv, hands::HandBone,
};
/// add debug renderer for controllers
@@ -264,97 +264,6 @@ pub fn spawn_hand_entities(mut commands: Commands) {
commands.insert_resource(hand_resource);
}
#[derive(Component, Debug, Clone, Copy)]
pub enum HandBone {
Palm,
Wrist,
ThumbMetacarpal,
ThumbProximal,
ThumbDistal,
ThumbTip,
IndexMetacarpal,
IndexProximal,
IndexIntermediate,
IndexDistal,
IndexTip,
MiddleMetacarpal,
MiddleProximal,
MiddleIntermediate,
MiddleDistal,
MiddleTip,
RingMetacarpal,
RingProximal,
RingIntermediate,
RingDistal,
RingTip,
LittleMetacarpal,
LittleProximal,
LittleIntermediate,
LittleDistal,
LittleTip,
}
impl HandBone {
pub const fn get_all_bones() -> [HandBone; 26] {
[
HandBone::Palm,
HandBone::Wrist,
HandBone::ThumbMetacarpal,
HandBone::ThumbProximal,
HandBone::ThumbDistal,
HandBone::ThumbTip,
HandBone::IndexMetacarpal,
HandBone::IndexProximal,
HandBone::IndexIntermediate,
HandBone::IndexDistal,
HandBone::IndexTip,
HandBone::MiddleMetacarpal,
HandBone::MiddleProximal,
HandBone::MiddleIntermediate,
HandBone::MiddleDistal,
HandBone::MiddleTip,
HandBone::RingMetacarpal,
HandBone::RingProximal,
HandBone::RingIntermediate,
HandBone::RingDistal,
HandBone::RingTip,
HandBone::LittleMetacarpal,
HandBone::LittleProximal,
HandBone::LittleIntermediate,
HandBone::LittleDistal,
HandBone::LittleTip,
]
}
pub fn get_index_from_bone(&self) -> usize {
match &self {
HandBone::Palm => 0,
HandBone::Wrist => 1,
HandBone::ThumbMetacarpal => 2,
HandBone::ThumbProximal => 3,
HandBone::ThumbDistal => 4,
HandBone::ThumbTip => 5,
HandBone::IndexMetacarpal => 6,
HandBone::IndexProximal => 7,
HandBone::IndexIntermediate => 8,
HandBone::IndexDistal => 9,
HandBone::IndexTip => 10,
HandBone::MiddleMetacarpal => 11,
HandBone::MiddleProximal => 12,
HandBone::MiddleIntermediate => 13,
HandBone::MiddleDistal => 14,
HandBone::MiddleTip => 15,
HandBone::RingMetacarpal => 16,
HandBone::RingProximal => 17,
HandBone::RingIntermediate => 18,
HandBone::RingDistal => 19,
HandBone::RingTip => 20,
HandBone::LittleMetacarpal => 21,
HandBone::LittleProximal => 22,
HandBone::LittleIntermediate => 23,
HandBone::LittleDistal => 24,
HandBone::LittleTip => 25,
}
}
}
pub fn update_hand_states(
oculus_controller: Res<OculusController>,

18
src/xr_input/hands/emulated.rs Normal file
View File

@@ -0,0 +1,18 @@
use bevy::prelude::*;
use crate::xr_input::Hand;
use super::HandBone;
#[derive(Deref, DerefMut, Resource)]
pub struct EmulatedHandPose(pub Box<dyn Fn(Hand, HandBone) -> (Vec3, Quat) + Send + Sync>);
pub struct EmulatedHandsPlugin;
impl Plugin for EmulatedHandsPlugin {
fn build(&self, app: &mut App) {
app.add_systems(Update, update_hand_skeleton_from_emulated);
}
}
pub(crate) fn update_hand_skeleton_from_emulated() {}

164
src/xr_input/hands/hand_tracking.rs Normal file
View File

@@ -0,0 +1,164 @@
use bevy::prelude::*;
use openxr::{HandTracker, Result, SpaceLocationFlags};
use crate::{
input::XrInput,
resources::{XrFrameState, XrSession},
xr_input::{
hand::HandBoneRadius, hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv,
Vec3Conv,
},
};
use super::BoneTrackingStatus;
#[derive(Resource, PartialEq)]
pub enum DisableHandTracking {
OnlyLeft,
OnlyRight,
Both,
}
pub struct HandTrackingPlugin;
#[derive(Resource)]
pub struct HandTrackingData {
left_hand: HandTracker,
right_hand: HandTracker,
}
impl HandTrackingData {
pub fn new(session: &XrSession) -> Result<HandTrackingData> {
let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
Ok(HandTrackingData {
left_hand: left,
right_hand: right,
})
}
pub fn get_ref<'a>(
&'a self,
input: &'a XrInput,
frame_state: &'a XrFrameState,
) -> HandTrackingRef<'a> {
HandTrackingRef {
tracking: self,
input,
frame_state,
}
}
}
pub struct HandTrackingRef<'a> {
tracking: &'a HandTrackingData,
input: &'a XrInput,
frame_state: &'a XrFrameState,
}
#[derive(Debug)]
pub struct HandJoint {
position: Vec3,
position_valid: bool,
position_tracked: bool,
orientaion: Quat,
orientaion_valid: bool,
orientaion_tracked: bool,
radius: f32,
}
pub struct HandJoints {
inner: [HandJoint; 26],
}
impl HandJoints {
pub fn get_joint(&self, bone: HandBone) -> &HandJoint {
&self.inner[bone.get_index_from_bone()]
}
}
impl<'a> HandTrackingRef<'a> {
pub fn get_poses(&self, side: Hand) -> Option<HandJoints> {
self.input
.stage
.locate_hand_joints(
match side {
Hand::Left => &self.tracking.left_hand,
Hand::Right => &self.tracking.right_hand,
},
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
.map(|joints| {
joints
.into_iter()
.map(|joint| HandJoint {
position: joint.pose.position.to_vec3(),
orientaion: joint.pose.orientation.to_quat(),
position_valid: joint
.location_flags
.contains(SpaceLocationFlags::POSITION_VALID),
position_tracked: joint
.location_flags
.contains(SpaceLocationFlags::POSITION_TRACKED),
orientaion_valid: joint
.location_flags
.contains(SpaceLocationFlags::ORIENTATION_VALID),
orientaion_tracked: joint
.location_flags
.contains(SpaceLocationFlags::ORIENTATION_TRACKED),
radius: joint.radius,
})
.collect::<Vec<HandJoint>>()
.try_into()
.unwrap()
})
.map(|joints| HandJoints { inner: joints })
}
}
impl Plugin for HandTrackingPlugin {
fn build(&self, app: &mut App) {
app.add_systems(
Update,
(update_hand_bones).run_if(|dh: Option<Res<DisableHandTracking>>| {
!dh.is_some_and(|v| *v == DisableHandTracking::Both)
}),
);
}
}
pub fn update_hand_bones(
hand_tracking: Res<HandTrackingData>,
xr_input: Res<XrInput>,
xr_frame_state: Res<XrFrameState>,
root_query: Query<(&Transform, With<OpenXRTrackingRoot>, Without<HandBone>)>,
mut bones: Query<(
&mut Transform,
&Hand,
&HandBone,
&mut HandBoneRadius,
&mut BoneTrackingStatus,
)>,
) {
let hand_ref = hand_tracking.get_ref(&xr_input, &xr_frame_state);
let (root_transform, _, _) = root_query.get_single().unwrap();
let left_hand_data = hand_ref.get_poses(Hand::Left);
let right_hand_data = hand_ref.get_poses(Hand::Right);
bones
.par_iter_mut()
.for_each(|(mut transform, hand, bone, mut radius, mut status)| {
let bone_data = match (hand, &left_hand_data, &right_hand_data) {
(Hand::Left, Some(data), _) => data.get_joint(*bone),
(Hand::Right, _, Some(data)) => data.get_joint(*bone),
_ => {
*status = BoneTrackingStatus::Emulated;
return;
}
};
if *status == BoneTrackingStatus::Emulated {
*status = BoneTrackingStatus::Tracked;
}
radius.0 = bone_data.radius;
*transform = transform
.with_translation(root_transform.transform_point(bone_data.position))
.with_rotation(root_transform.rotation * bone_data.orientaion)
});
}

108
src/xr_input/hands/mod.rs Normal file
View File

@@ -0,0 +1,108 @@
use bevy::prelude::*;
pub mod emulated;
pub mod hand_tracking;
pub struct HandsPlugin;
impl Plugin for HandsPlugin {
fn build(&self, app: &mut bevy::prelude::App) {}
}
#[derive(Component, Debug, Clone, Copy, PartialEq)]
pub enum BoneTrackingStatus {
Emulated,
Tracked,
}
#[derive(Component, Debug, Clone, Copy)]
pub enum HandBone {
Palm,
Wrist,
ThumbMetacarpal,
ThumbProximal,
ThumbDistal,
ThumbTip,
IndexMetacarpal,
IndexProximal,
IndexIntermediate,
IndexDistal,
IndexTip,
MiddleMetacarpal,
MiddleProximal,
MiddleIntermediate,
MiddleDistal,
MiddleTip,
RingMetacarpal,
RingProximal,
RingIntermediate,
RingDistal,
RingTip,
LittleMetacarpal,
LittleProximal,
LittleIntermediate,
LittleDistal,
LittleTip,
}
impl HandBone {
pub const fn get_all_bones() -> [HandBone; 26] {
[
HandBone::Palm,
HandBone::Wrist,
HandBone::ThumbMetacarpal,
HandBone::ThumbProximal,
HandBone::ThumbDistal,
HandBone::ThumbTip,
HandBone::IndexMetacarpal,
HandBone::IndexProximal,
HandBone::IndexIntermediate,
HandBone::IndexDistal,
HandBone::IndexTip,
HandBone::MiddleMetacarpal,
HandBone::MiddleProximal,
HandBone::MiddleIntermediate,
HandBone::MiddleDistal,
HandBone::MiddleTip,
HandBone::RingMetacarpal,
HandBone::RingProximal,
HandBone::RingIntermediate,
HandBone::RingDistal,
HandBone::RingTip,
HandBone::LittleMetacarpal,
HandBone::LittleProximal,
HandBone::LittleIntermediate,
HandBone::LittleDistal,
HandBone::LittleTip,
]
}
pub fn get_index_from_bone(&self) -> usize {
match &self {
HandBone::Palm => 0,
HandBone::Wrist => 1,
HandBone::ThumbMetacarpal => 2,
HandBone::ThumbProximal => 3,
HandBone::ThumbDistal => 4,
HandBone::ThumbTip => 5,
HandBone::IndexMetacarpal => 6,
HandBone::IndexProximal => 7,
HandBone::IndexIntermediate => 8,
HandBone::IndexDistal => 9,
HandBone::IndexTip => 10,
HandBone::MiddleMetacarpal => 11,
HandBone::MiddleProximal => 12,
HandBone::MiddleIntermediate => 13,
HandBone::MiddleDistal => 14,
HandBone::MiddleTip => 15,
HandBone::RingMetacarpal => 16,
HandBone::RingProximal => 17,
HandBone::RingIntermediate => 18,
HandBone::RingDistal => 19,
HandBone::RingTip => 20,
HandBone::LittleMetacarpal => 21,
HandBone::LittleProximal => 22,
HandBone::LittleIntermediate => 23,
HandBone::LittleDistal => 24,
HandBone::LittleTip => 25,
}
}
}

28
src/xr_input/handtracking.rs
View File

@@ -1,15 +1,11 @@
use std::mem::MaybeUninit;
use bevy::prelude::*;
use openxr::{HandJointLocationEXT, HandTracker, Result};
use crate::{
input::XrInput,
resources::{XrFrameState, XrFrameWaiter, XrSession},
resources::{XrFrameState, XrSession},
};
use super::hand::HandBone;
#[derive(Resource)]
pub struct HandTrackingTracker {
left_hand: HandTracker,
@@ -44,10 +40,8 @@ pub struct HandTrackingRef<'a> {
frame_state: &'a XrFrameState,
}
// pub type HandJoints = [(HandJointLocationEXT, HandBone); 26];
impl<'a> HandTrackingRef<'a> {
pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT;26]> {
pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
self.input
.stage
.locate_hand_joints(
@@ -55,16 +49,8 @@ impl<'a> HandTrackingRef<'a> {
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
// .map(|joints| {
// joints
// .into_iter()
// .zip(HandBone::get_all_bones().into_iter())
// .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
// .try_into()
// .unwrap()
// })
}
pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT;26]> {
pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
self.input
.stage
.locate_hand_joints(
@@ -72,13 +58,5 @@ impl<'a> HandTrackingRef<'a> {
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
// .map(|joints| {
// joints
// .into_iter()
// .zip(HandBone::get_all_bones().into_iter())
// .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
// .try_into()
// .unwrap()
// })
}
}

1
src/xr_input/mod.rs
View File

@@ -8,6 +8,7 @@ pub mod xr_camera;
pub mod hand_poses;
pub mod hand;
pub mod handtracking;
pub mod hands;
use crate::resources::XrSession;
use crate::xr_begin_frame;