Avii/bevy_oxr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Done

Browse Source
This commit is contained in:
Schmarni
2023-11-16 06:09:36 +01:00
parent 851c7b7587
commit 5b443c3b35
6 changed files with 189 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/demo/src/lib.rs
View File

@@ -29,7 +29,7 @@ use bevy_oxr::{
oculus_touch::OculusController,
prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig},
trackers::{OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker},
Hand,
Hand, actions::XrActionSets,
},
DefaultXrPlugins,
};

14
src/lib.rs
View File

@@ -23,12 +23,10 @@ use openxr as xr;
use resources::*;
use xr_input::controllers::XrControllerType;
use xr_input::hands::emulated::EmulatedHandsPlugin;
use xr_input::hands::hand_tracking::HandTrackingPlugin;
use xr_input::hands::hand_tracking::{HandTrackingPlugin, HandTrackingData};
use xr_input::handtracking::HandTrackingTracker;
use xr_input::OpenXrInput;
use crate::xr_input::oculus_touch::ActionSets;
const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
pub const LEFT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(1208214591);
@@ -91,7 +89,7 @@ impl Plugin for OpenXrPlugin {
views,
frame_state,
hand_tracking_enabled,
) = graphics::initialize_xr_graphics(primary_window,self.0).unwrap();
) = graphics::initialize_xr_graphics(primary_window, self.0).unwrap();
// std::thread::sleep(Duration::from_secs(5));
debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
debug!("Configured wgpu adapter Features: {:#?}", device.features());
@@ -163,10 +161,9 @@ impl Plugin for OpenXrPlugin {
.insert_resource(action_sets.clone());
let hands = xr_instance.exts().ext_hand_tracking.is_some();
if hands {
app.insert_resource(HandTrackingTracker::new(&session).unwrap());
app.insert_resource(HandInputSource::OpenXr);
app.insert_resource(HandTrackingData::new(&session).unwrap());
} else {
app.insert_resource(HandInputSource::Emulated);
app.insert_resource(DisableHandTracking::Both);
}
let (left, right) = swapchain.get_render_views();
@@ -224,7 +221,8 @@ impl PluginGroup for DefaultXrPlugins {
.disable::<PipelinedRenderingPlugin>()
.add_before::<RenderPlugin, _>(OpenXrPlugin::default())
.add_after::<OpenXrPlugin, _>(OpenXrInput::new(XrControllerType::OculusTouch))
.add(EmulatedHandsPlugin).add(HandTrackingPlugin)
.add(EmulatedHandsPlugin)
.add(HandTrackingPlugin)
.set(WindowPlugin {
#[cfg(not(target_os = "android"))]
primary_window: Some(Window {

26
src/xr_input/debug_gizmos.rs
View File

@@ -17,7 +17,7 @@ use super::{
actions::XrActionSets,
handtracking::{HandTrackingRef, HandTrackingTracker},
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
QuatConv,
QuatConv, hands::hand_tracking::HandTrackingData,
};
/// add debug renderer for controllers
@@ -55,19 +55,16 @@ pub fn draw_gizmos(
Without<OpenXRLeftController>,
Without<OpenXRTrackingRoot>,
)>,
hand_tracking: Option<Res<HandTrackingTracker>>,
hand_tracking: Option<Res<HandTrackingData>>,
action_sets: Res<XrActionSets>,
) {
if let Some(hand_tracking) = hand_tracking {
let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
if let Some(joints) = handtracking_ref.get_left_poses() {
for joint in joints {
let p = joint.pose.position;
let r = joint.pose.orientation;
let quat = r.to_quat();
let trans = Transform::from_rotation(quat);
if let Some(joints) = handtracking_ref.get_poses(Hand::Left) {
for joint in joints.inner() {
let trans = Transform::from_rotation(joint.orientation);
gizmos.circle(
(p.x, p.y, p.z).into(),
joint.position,
trans.forward(),
joint.radius,
Color::ORANGE_RED,
@@ -76,14 +73,11 @@ pub fn draw_gizmos(
} else {
info!("left_hand_poses returned None");
}
if let Some(joints) = handtracking_ref.get_right_poses() {
for joint in joints {
let p = joint.pose.position;
let r = joint.pose.orientation;
let quat = r.to_quat();
let trans = Transform::from_rotation(quat);
if let Some(joints) = handtracking_ref.get_poses(Hand::Right) {
for joint in joints.inner() {
let trans = Transform::from_rotation(joint.orientation);
gizmos.circle(
(p.x, p.y, p.z).into(),
joint.position,
trans.forward(),
joint.radius,
Color::LIME_GREEN,

13
src/xr_input/hand.rs
View File

@@ -16,10 +16,11 @@ use crate::{
use super::{
actions::XrActionSets,
hand_poses::get_simulated_open_hand_transforms,
hands::{BoneTrackingStatus, HandBone},
handtracking::HandTrackingTracker,
oculus_touch::OculusController,
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker, OpenXRTrackingRoot},
Hand, QuatConv, hands::{HandBone, BoneTrackingStatus},
Hand, QuatConv,
};
/// add debug renderer for controllers
@@ -198,6 +199,7 @@ pub fn spawn_hand_entities(mut commands: Commands) {
OpenXRTracker,
hand.clone(),
BoneTrackingStatus::Emulated,
HandBoneRadius(1.0),
))
.id();
match hand {
@@ -267,7 +269,6 @@ pub fn spawn_hand_entities(mut commands: Commands) {
commands.insert_resource(hand_resource);
}
pub fn update_hand_states(
oculus_controller: Res<OculusController>,
hand_states_option: Option<ResMut<HandStatesResource>>,
@@ -1063,20 +1064,20 @@ pub struct HandBoneRadius(pub f32);
pub fn draw_hand_entities(
mut gizmos: Gizmos,
query: Query<(&Transform, &HandBone, Option<&HandBoneRadius>)>,
query: Query<(&Transform, &HandBone, &HandBoneRadius)>,
) {
for (transform, hand_bone, hand_bone_radius) in query.iter() {
let (radius, color) = get_bone_gizmo_style(hand_bone);
let (_, color) = get_bone_gizmo_style(hand_bone);
gizmos.sphere(
transform.translation,
transform.rotation,
hand_bone_radius.map_or(radius, |r| r.0),
hand_bone_radius.0,
color,
);
}
}
fn get_bone_gizmo_style(hand_bone: &HandBone) -> (f32, Color) {
pub(crate) fn get_bone_gizmo_style(hand_bone: &HandBone) -> (f32, Color) {
match hand_bone {
HandBone::Palm => (0.01, Color::WHITE),
HandBone::Wrist => (0.01, Color::GRAY),

279
src/xr_input/hands/emulated.rs
View File

@@ -6,15 +6,19 @@ use openxr::{Action, ActionTy, Binding, HandJoint};
use crate::{
resources::{XrInstance, XrSession},
xr_input::{
actions::{
ActionHandednes, ActionType, SetupActionSet, SetupActionSets, XrActionSets, XrBinding,
},
controllers::Touchable,
hand::{get_bone_gizmo_style, HandBoneRadius},
hand_poses::get_simulated_open_hand_transforms,
oculus_touch::ActionSets,
trackers::{OpenXRLeftController, OpenXRRightController},
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
Hand, InteractionProfileBindings,
},
};
use super::HandBone;
use super::{BoneTrackingStatus, HandBone};
pub enum TouchValue<T: ActionTy> {
None,
@@ -32,152 +36,119 @@ pub struct EmulatedHandsPlugin;
impl Plugin for EmulatedHandsPlugin {
fn build(&self, app: &mut App) {
app.add_systems(PreUpdate, update_hand_skeleton_from_emulated);
app.add_systems(
Startup,
setup_hand_emulation_action_set.map(|res| res.unwrap()),
);
app.add_systems(Startup, setup_hand_emulation_action_set);
}
}
#[derive(Resource)]
pub struct HandEmulationActionSet {
thumb_touch: Action<bool>,
thumb_x: Action<f32>,
thumb_y: Action<f32>,
index: Touchable<f32>,
middle: Touchable<f32>,
ring: Touchable<f32>,
little: Touchable<f32>,
}
fn setup_hand_emulation_action_set(
instance: Res<XrInstance>,
session: Res<XrSession>,
mut action_sets: ResMut<ActionSets>,
mut commands: Commands,
mut bindings: ResMut<InteractionProfileBindings>
) -> anyhow::Result<()> {
let left_path = instance.string_to_path("/user/hand/left").unwrap();
let right_path = instance.string_to_path("/user/hand/right").unwrap();
let hands = [left_path, right_path];
// This unwrap Should not trigger since both strings are not empty
let action_set = instance
.create_action_set("hand_pose_approximation_set", "Hand Pose Approximaiton", 0)
.unwrap();
let hand_action_set = HandEmulationActionSet {
thumb_touch: action_set.create_action::<bool>("thumb_touch", "Thumb Touched", &hands)?,
thumb_x: action_set.create_action::<f32>("thumb_x", "Thumb X", &hands)?,
thumb_y: action_set.create_action::<f32>("thumb_y", "Thumb Y", &hands)?,
const HAND_ACTION_SET: &'static str = "hand_pose_approx";
index: Touchable::<f32> {
inner: action_set.create_action("index_value", "Index Finger Pull", &hands)?,
touch: action_set.create_action("index_touch", "Index Finger Touch", &hands)?,
},
middle: Touchable::<f32> {
inner: action_set.create_action("middle_value", "Middle Finger Pull", &hands)?,
touch: action_set.create_action("middle_touch", "Middle Finger Touch", &hands)?,
},
ring: Touchable::<f32> {
inner: action_set.create_action("ring_value", "Ring Finger Pull", &hands)?,
touch: action_set.create_action("ring_touch", "Ring Finger Touch", &hands)?,
},
little: Touchable::<f32> {
inner: action_set.create_action("little_value", "Little Finger Pull", &hands)?,
touch: action_set.create_action("little_touch", "Little Finger Touch", &hands)?,
},
};
fn setup_hand_emulation_action_set(mut action_sets: ResMut<SetupActionSets>) {
let mut action_set = action_sets.add_action_set(HAND_ACTION_SET, "Hand Pose Approximaiton", 0);
action_set.new_action(
"thumb_touch",
"Thumb Touched",
ActionType::Bool,
ActionHandednes::Double,
);
action_set.new_action(
"thumb_x",
"Thumb X",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"thumb_y",
"Thumb Y",
ActionType::F32,
ActionHandednes::Double,
);
suggest_oculus_touch_profile(&instance, &hand_action_set,bindings)?;
action_set.new_action(
"index_touch",
"Index Finger Touched",
ActionType::Bool,
ActionHandednes::Double,
);
action_set.new_action(
"index_value",
"Index Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
session.attach_action_sets(&[&action_set])?;
action_set.new_action(
"middle_value",
"Middle Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"ring_value",
"Ring Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"little_value",
"Little Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
action_sets.0.push(action_set);
commands.insert_resource(hand_action_set);
Ok(())
suggest_oculus_touch_profile(&mut action_set);
}
pub struct EmulatedHandPoseData {}
fn bind<'a, T: ActionTy>(
action: &'a Action<T>,
path: &str,
i: &XrInstance,
bindings: &mut Vec<Binding<'a>>,
) -> anyhow::Result<()> {
bindings.push(Binding::new(
&action,
i.string_to_path(&("/user/hand/left/input".to_string() + path))?,
));
bindings.push(Binding::new(
&action,
i.string_to_path(&("/user/hand/right/input".to_string() + path))?,
));
Ok(())
}
fn bind_single<'a, T: ActionTy>(
action: &'a Action<T>,
path: &str,
hand: Hand,
i: &XrInstance,
bindings: &mut Vec<Binding<'a>>,
) -> anyhow::Result<()> {
match hand {
Hand::Left => bindings.push(Binding::new(
&action,
i.string_to_path(&("/user/hand/left/input".to_string() + path))?,
)),
Hand::Right => bindings.push(Binding::new(
&action,
i.string_to_path(&("/user/hand/right/input".to_string() + path))?,
)),
}
Ok(())
}
fn suggest_oculus_touch_profile(
i: &XrInstance,
action_set: &HandEmulationActionSet,
mut bindings: ResMut<InteractionProfileBindings>
) -> anyhow::Result<()> {
let mut b = bindings.entry("/interaction_profiles/oculus/touch_controller").or_default();
bind(&action_set.thumb_x, "/thumbstick/x", i, &mut b)?;
bind(&action_set.thumb_y, "/thumbstick/y", i, &mut b)?;
bind(&action_set.thumb_touch, "/thumbstick/touch", i, &mut b)?;
bind(&action_set.thumb_touch, "/thumbrest/touch", i, &mut b)?;
// bind_single(&action_set.thumb_touch, "/x/touch", Hand::Left, i, &mut b)?;
// bind_single(&action_set.thumb_touch, "/y/touch", Hand::Left, i, &mut b)?;
// bind_single(&action_set.thumb_touch, "/a/touch", Hand::Right, i, &mut b)?;
// bind_single(&action_set.thumb_touch, "/b/touch", Hand::Right, i, &mut b)?;
// bind(&action_set.index.touch, "/trigger/touch", i, &mut b)?;
// bind(&action_set.index.inner, "/trigger/value", i, &mut b)?;
//
// bind(&action_set.middle.touch, "/squeeze/touch", i, &mut b)?;
// bind(&action_set.middle.inner, "/squeeze/value", i, &mut b)?;
// bind(&action_set.ring.touch, "/squeeze/touch", i, &mut b)?;
// bind(&action_set.ring.inner, "/squeeze/value", i, &mut b)?;
// bind(&action_set.little.touch, "/squeeze/touch", i, &mut b)?;
// bind(&action_set.little.inner, "/squeeze/value", i, &mut b)?;
i.suggest_interaction_profile_bindings(
i.string_to_path("/interaction_profiles/oculus/touch_controller")?,
&b,
)?;
Ok(())
fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) {
action_set.suggest_binding(
"/interaction_profiles/oculus/touch_controller",
&[
XrBinding::new("thumb_x", "/user/hand/left/input/thumbstick/x"),
XrBinding::new("thumb_x", "/user/hand/right/input/thumbstick/x"),
XrBinding::new("thumb_y", "/user/hand/left/input/thumbstick/y"),
XrBinding::new("thumb_y", "/user/hand/right/input/thumbstick/y"),
XrBinding::new("thumb_touch", "/user/hand/left/input/thumbstick/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/thumbstick/touch"),
XrBinding::new("thumb_touch", "/user/hand/left/input/x/touch"),
XrBinding::new("thumb_touch", "/user/hand/left/input/y/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/a/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/b/touch"),
XrBinding::new("thumb_touch", "/user/hand/left/input/thumbrest/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/thumbrest/touch"),
XrBinding::new("index_touch", "/user/hand/left/input/trigger/touch"),
XrBinding::new("index_value", "/user/hand/left/input/trigger/value"),
XrBinding::new("index_touch", "/user/hand/right/input/trigger/touch"),
XrBinding::new("index_value", "/user/hand/right/input/trigger/value"),
XrBinding::new("middle_value", "/user/hand/left/input/squeeze/value"),
XrBinding::new("middle_value", "/user/hand/right/input/squeeze/value"),
XrBinding::new("ring_value", "/user/hand/left/input/squeeze/value"),
XrBinding::new("ring_value", "/user/hand/right/input/squeeze/value"),
XrBinding::new("little_value", "/user/hand/left/input/squeeze/value"),
XrBinding::new("little_value", "/user/hand/right/input/squeeze/value"),
],
);
}
pub(crate) fn update_hand_skeleton_from_emulated(
session: Res<XrSession>,
instance: Res<XrInstance>,
action_set: Res<HandEmulationActionSet>,
action_sets: Res<XrActionSets>,
left_controller_transform: Query<&Transform, With<OpenXRLeftController>>,
right_controller_transform: Query<&Transform, With<OpenXRRightController>>,
tracking_root_transform: Query<&Transform, With<OpenXRTrackingRoot>>,
mut bones: Query<
(&mut Transform, &HandBone, &Hand),
(
&mut Transform,
&HandBone,
&Hand,
&BoneTrackingStatus,
&mut HandBoneRadius,
),
(
Without<OpenXRLeftController>,
Without<OpenXRRightController>,
Without<OpenXRTrackingRoot>,
),
>,
) {
@@ -192,8 +163,9 @@ pub(crate) fn update_hand_skeleton_from_emulated(
Hand::Right,
),
] {
let thumb_curl = match action_set
.thumb_touch
let thumb_curl = match action_sets
.get_action_bool(HAND_ACTION_SET, "thumb_touch")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state
@@ -201,27 +173,27 @@ pub(crate) fn update_hand_skeleton_from_emulated(
true => 1.0,
false => 0.0,
};
let index_curl = action_set
.index
.inner
let index_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "index_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
let middle_curl = action_set
.middle
.inner
let middle_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "middle_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
let ring_curl = action_set
.ring
.inner
let ring_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "ring_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
let little_curl = action_set
.little
.inner
let little_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "little_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
@@ -241,11 +213,21 @@ pub(crate) fn update_hand_skeleton_from_emulated(
little_curl,
);
}
for (mut t, bone, hand) in bones.iter_mut() {
let trt = tracking_root_transform.single();
for (mut t, bone, hand, status, mut radius) in bones.iter_mut() {
match status {
BoneTrackingStatus::Emulated => {}
BoneTrackingStatus::Tracked => continue,
}
radius.0 = get_bone_gizmo_style(bone).0;
*t = data[match hand {
Hand::Left => 0,
Hand::Right => 1,
}][bone.get_index_from_bone()]
}][bone.get_index_from_bone()];
*t = t.with_scale(trt.scale);
*t = t.with_rotation(trt.rotation * t.rotation);
*t = t.with_translation(trt.transform_point(t.translation));
}
}
pub fn update_hand_bones_emulated(
@@ -537,6 +519,15 @@ pub fn update_hand_bones_emulated(
calc_transforms
}
fn get_bone_curl_angle(bone: HandJoint, thumb_curl: f32) -> f32 {
todo!()
fn get_bone_curl_angle(bone: HandJoint, curl: f32) -> f32 {
let mul: f32 = match bone {
HandJoint::INDEX_PROXIMAL => 0.0,
HandJoint::MIDDLE_PROXIMAL => 0.0,
HandJoint::RING_PROXIMAL => 0.0,
HandJoint::LITTLE_PROXIMAL => 0.0,
HandJoint::THUMB_PROXIMAL => 0.0,
_ => 1.0,
};
let curl_angle = -((mul * curl * 80.0) + 5.0);
return curl_angle;
}

45
src/xr_input/hands/hand_tracking.rs
View File

@@ -55,18 +55,23 @@ pub struct HandTrackingRef<'a> {
}
#[derive(Debug)]
pub struct HandJoint {
position: Vec3,
position_valid: bool,
position_tracked: bool,
orientaion: Quat,
orientaion_valid: bool,
orientaion_tracked: bool,
radius: f32,
pub position: Vec3,
pub position_valid: bool,
pub position_tracked: bool,
pub orientation: Quat,
pub orientation_valid: bool,
pub orientation_tracked: bool,
pub radius: f32,
}
pub struct HandJoints {
inner: [HandJoint; 26],
}
impl HandJoints {
pub fn inner(&self) -> &[HandJoint; 26] {
&self.inner
}
}
impl HandJoints {
pub fn get_joint(&self, bone: HandBone) -> &HandJoint {
@@ -91,17 +96,17 @@ impl<'a> HandTrackingRef<'a> {
.into_iter()
.map(|joint| HandJoint {
position: joint.pose.position.to_vec3(),
orientaion: joint.pose.orientation.to_quat(),
orientation: joint.pose.orientation.to_quat(),
position_valid: joint
.location_flags
.contains(SpaceLocationFlags::POSITION_VALID),
position_tracked: joint
.location_flags
.contains(SpaceLocationFlags::POSITION_TRACKED),
orientaion_valid: joint
orientation_valid: joint
.location_flags
.contains(SpaceLocationFlags::ORIENTATION_VALID),
orientaion_tracked: joint
orientation_tracked: joint
.location_flags
.contains(SpaceLocationFlags::ORIENTATION_TRACKED),
radius: joint.radius,
@@ -126,8 +131,8 @@ impl Plugin for HandTrackingPlugin {
}
pub fn update_hand_bones(
disabled_tracking: Res<DisableHandTracking>,
hand_tracking: Res<HandTrackingData>,
disabled_tracking: Option<Res<DisableHandTracking>>,
hand_tracking: Option<Res<HandTrackingData>>,
xr_input: Res<XrInput>,
xr_frame_state: Res<XrFrameState>,
root_query: Query<(&Transform, With<OpenXRTrackingRoot>, Without<HandBone>)>,
@@ -139,19 +144,25 @@ pub fn update_hand_bones(
&mut BoneTrackingStatus,
)>,
) {
let hand_ref = hand_tracking.get_ref(&xr_input, &xr_frame_state);
let hand_ref = match hand_tracking.as_ref() {
Some(h) => h.get_ref(&xr_input, &xr_frame_state),
None => {
warn!("No Handtracking data!");
return;
}
};
let (root_transform, _, _) = root_query.get_single().unwrap();
let left_hand_data = hand_ref.get_poses(Hand::Left);
let right_hand_data = hand_ref.get_poses(Hand::Right);
bones
.par_iter_mut()
.for_each(|(mut transform, hand, bone, mut radius, mut status)| {
match (&hand, disabled_tracking.as_ref()) {
(Hand::Left, DisableHandTracking::OnlyLeft) => {
match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) {
(Hand::Left, Some(DisableHandTracking::OnlyLeft)) => {
*status = BoneTrackingStatus::Emulated;
return;
}
(Hand::Right, DisableHandTracking::OnlyRight) => {
(Hand::Right, Some(DisableHandTracking::OnlyRight)) => {
*status = BoneTrackingStatus::Emulated;
return;
}
@@ -171,6 +182,6 @@ pub fn update_hand_bones(
radius.0 = bone_data.radius;
*transform = transform
.with_translation(root_transform.transform_point(bone_data.position))
.with_rotation(root_transform.rotation * bone_data.orientaion)
.with_rotation(root_transform.rotation * bone_data.orientation)
});
}