Avii/bevy_oxr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

handtracking not working on quest 2 v62

Browse Source
This commit is contained in:
Schmarni
2024-02-15 06:04:16 +01:00
parent 3803968b7c
commit 31d7b05b4a
9 changed files with 163 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
examples/android/src/lib.rs
View File

@@ -6,6 +6,7 @@ use bevy_oxr::graphics::XrAppInfo;
use bevy_oxr::passthrough::{PausePassthrough, ResumePassthrough, XrPassthroughState};
use bevy_oxr::xr_init::xr_only;
use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer;
use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer;
use bevy_oxr::xr_input::hands::HandBone;
use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig};
use bevy_oxr::xr_input::trackers::{
@@ -17,6 +18,7 @@ use bevy_oxr::DefaultXrPlugins;
fn main() {
let mut xr_extensions = XrExtensions::default();
xr_extensions.enable_fb_passthrough();
xr_extensions.enable_hand_tracking();
App::new()
.add_plugins(DefaultXrPlugins {
reqeusted_extensions: xr_extensions,
@@ -28,8 +30,13 @@ fn main() {
// .add_plugins(OpenXrDebugRenderer)
.add_plugins(LogDiagnosticsPlugin::default())
.add_plugins(FrameTimeDiagnosticsPlugin)
.add_plugins(HandInputDebugRenderer)
.add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup)
.add_systems(Startup, setup)
.add_systems(Update, (proto_locomotion, toggle_passthrough).run_if(xr_only()))
.add_systems(
Update,
(proto_locomotion, toggle_passthrough).run_if(xr_only()),
)
.add_systems(Update, debug_hand_render.run_if(xr_only()))
.add_systems(Startup, spawn_controllers_example)
.insert_resource(PrototypeLocomotionConfig::default())

17
src/lib.rs
View File

@@ -6,12 +6,8 @@ pub mod resources;
pub mod xr_init;
pub mod xr_input;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Mutex};
use crate::passthrough::ResumePassthrough;
use crate::xr_init::{StartXrSession, XrInitPlugin};
use crate::xr_input::oculus_touch::ActionSets;
use bevy::app::{AppExit, PluginGroupBuilder};
@@ -34,10 +30,12 @@ use xr_init::{
xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, SetupXrData, XrEarlyInitPlugin,
XrHasWaited, XrShouldRender, XrStatus,
};
use xr_input::actions::OpenXrActionsPlugin;
use xr_input::controllers::XrControllerType;
use xr_input::hands::emulated::HandEmulationPlugin;
use xr_input::hands::hand_tracking::HandTrackingPlugin;
use xr_input::hands::HandPlugin;
use xr_input::xr_camera::XrCameraPlugin;
use xr_input::OpenXrInput;
const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
@@ -81,7 +79,7 @@ impl Plugin for OpenXrPlugin {
debug!("Configured wgpu adapter Features: {:#?}", device.features());
warn!("Starting with OpenXR Instance");
app.insert_resource(xr_instance.clone());
app.insert_resource(blend_mode.clone());
app.insert_resource(blend_mode);
app.insert_resource(ActionSets(vec![]));
app.insert_resource(xr_instance);
app.insert_resource(blend_mode);
@@ -120,7 +118,9 @@ impl Plugin for OpenXrPlugin {
(
xr_reset_per_frame_resources,
xr_wait_frame.run_if(xr_only()),
// xr_begin_frame.run_if(xr_only()),
locate_views.run_if(xr_only()),
apply_deferred,
)
.chain()
@@ -215,8 +215,10 @@ impl PluginGroup for DefaultXrPlugins {
reqeusted_extensions: self.reqeusted_extensions,
app_info: self.app_info.clone(),
})
.add_after::<OpenXrPlugin, _>(OpenXrInput::new(XrControllerType::OculusTouch))
.add_after::<OpenXrPlugin, _>(XrInitPlugin)
.add(XrInitPlugin)
.add(OpenXrInput::new(XrControllerType::OculusTouch))
.add(OpenXrActionsPlugin)
.add(XrCameraPlugin)
.add_before::<OpenXrPlugin, _>(XrEarlyInitPlugin)
.add(HandPlugin)
.add(HandTrackingPlugin)
@@ -318,6 +320,7 @@ pub fn xr_wait_frame(
return;
}
};
// frame_state.predicted_display_time = xr::Time::from_nanos(frame_state.predicted_display_time.as_nanos() + frame_state.predicted_display_period.as_nanos());
**should_render = frame_state.should_render;
**waited = true;
}

1
src/resources.rs
View File

@@ -16,7 +16,6 @@ use core::ptr;
use openxr as xr;
xr_resource_wrapper!(XrInstance, xr::Instance);
// xr_resource_wrapper!(XrSession, xr::Session<xr::AnyGraphics>);
xr_resource_wrapper_copy!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode);
xr_resource_wrapper_copy!(XrResolution, UVec2);
xr_resource_wrapper_copy!(XrFormat, wgpu::TextureFormat);

4
src/xr_init/mod.rs
View File

@@ -54,6 +54,7 @@ pub fn xr_after_wait_only() -> impl FnMut(Res<XrHasWaited>) -> bool {
impl Plugin for XrEarlyInitPlugin {
fn build(&self, app: &mut App) {
add_schedules(app);
app.add_event::<SetupXrData>()
.add_event::<CleanupXrData>()
.add_event::<StartXrSession>()
@@ -63,7 +64,6 @@ impl Plugin for XrEarlyInitPlugin {
impl Plugin for XrInitPlugin {
fn build(&self, app: &mut App) {
add_schedules(app);
app.add_plugins(ExtractResourcePlugin::<XrStatus>::default());
app.add_plugins(ExtractResourcePlugin::<XrShouldRender>::default());
app.add_plugins(ExtractResourcePlugin::<XrHasWaited>::default());
@@ -106,7 +106,9 @@ fn setup_manual_texture_views(
}
pub fn setup_xr(world: &mut World) {
info!("Pre XrPreSetup");
world.run_schedule(XrPreSetup);
info!("Post XrPreSetup");
world.run_schedule(XrSetup);
world.run_schedule(XrPrePostSetup);
world.run_schedule(XrPostSetup);

41
src/xr_input/actions.rs
View File

@@ -6,7 +6,7 @@ use xr::{Action, Binding, Haptic, Posef, Vector2f};
use crate::{
resources::{XrInstance, XrSession},
xr_init::{XrCleanup, XrPrePostSetup, XrPreSetup},
xr_init::{xr_only, XrCleanup, XrPrePostSetup, XrPreSetup},
};
use super::oculus_touch::ActionSets;
@@ -16,13 +16,18 @@ pub use xr::sys::NULL_PATH;
pub struct OpenXrActionsPlugin;
impl Plugin for OpenXrActionsPlugin {
fn build(&self, app: &mut App) {
app.add_systems(XrPreSetup, insert_setup_action_sets);
app.add_systems(PreUpdate, sync_actions.run_if(xr_only()));
app.add_systems(
XrPreSetup,
(insert_setup_action_sets, apply_deferred).chain(),
);
app.add_systems(XrPrePostSetup, setup_oxr_actions);
app.add_systems(XrCleanup, clean_actions);
}
}
fn insert_setup_action_sets(mut cmds: Commands) {
info!("WHAT?!");
cmds.insert_resource(SetupActionSets {
sets: HashMap::new(),
});
@@ -57,7 +62,6 @@ pub fn setup_oxr_actions(world: &mut World) {
let right_path = instance.string_to_path("/user/hand/right").unwrap();
let hands = [left_path, right_path];
let mut oxr_action_sets = Vec::new();
let mut action_sets = XrActionSets { sets: default() };
// let mut action_bindings: HashMap<&'static str, Vec<xr::Path>> = HashMap::new();
let mut action_bindings: HashMap<
@@ -101,11 +105,11 @@ pub fn setup_oxr_actions(world: &mut World) {
}
}
}
oxr_action_sets.push(oxr_action_set);
// oxr_action_sets.push(oxr_action_set);
action_sets.sets.insert(
set_name,
ActionSet {
// oxr_action_set,
oxr_action_set,
actions,
enabled: true,
},
@@ -152,10 +156,15 @@ pub fn setup_oxr_actions(world: &mut World) {
.expect("Unable to suggest interaction bindings!");
}
session
.attach_action_sets(&oxr_action_sets.iter().collect::<Vec<_>>())
.attach_action_sets(
&action_sets
.sets
.values()
.map(|set| &set.oxr_action_set)
.collect::<Vec<_>>(),
)
.expect("Unable to attach action sets!");
world.insert_resource(ActionSets(oxr_action_sets));
world.insert_resource(action_sets);
}
@@ -267,6 +276,7 @@ pub struct ActionSet {
// add functionality to enable/disable action sets
enabled: bool,
actions: HashMap<&'static str, TypedAction>,
oxr_action_set: xr::ActionSet,
}
#[derive(Resource)]
@@ -380,3 +390,20 @@ impl XrActionSets {
}
}
}
pub fn sync_actions(action_sets: Res<XrActionSets>, session: Res<XrSession>) {
let active_sets = action_sets
.sets
.values()
.filter_map(|set| {
if set.enabled {
Some(xr::ActiveActionSet::new(&set.oxr_action_set))
} else {
None
}
})
.collect::<Vec<_>>();
if let Err(err) = session.sync_actions(&active_sets) {
warn!("OpenXR action sync error: {}", err);
}
}

2
src/xr_input/hands/common.rs
View File

@@ -170,7 +170,7 @@ pub fn spawn_hand_entities(mut commands: Commands) {
*bone,
OpenXRTracker,
*hand,
BoneTrackingStatus::Emulated,
BoneTrackingStatus::Tracked,
HandBoneRadius(0.1),
))
.id();

12
src/xr_input/hands/hand_tracking.rs
View File

@@ -63,6 +63,7 @@ pub struct HandJoint {
pub radius: f32,
}
#[derive(Debug)]
pub struct HandJoints {
inner: [HandJoint; 26],
}
@@ -175,9 +176,15 @@ pub fn update_hand_bones(
};
let left_hand_data = hand_ref.get_poses(Hand::Left);
let right_hand_data = hand_ref.get_poses(Hand::Right);
if left_hand_data.is_none() || right_hand_data.is_none() {
error!("something is very wrong for hand_tracking!! doesn't have data for both hands!");
}
info!("hand_tracking");
bones
.par_iter_mut()
.for_each(|(mut transform, hand, bone, mut radius, mut status)| {
info!("hand_tracking bone before filter");
match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) {
(Hand::Left, Some(DisableHandTracking::OnlyLeft)) => {
*status = BoneTrackingStatus::Emulated;
@@ -189,14 +196,17 @@ pub fn update_hand_bones(
}
_ => {}
}
info!("hand_tracking bone mid filter");
let bone_data = match (hand, &left_hand_data, &right_hand_data) {
(Hand::Left, Some(data), _) => data.get_joint(*bone),
(Hand::Right, _, Some(data)) => data.get_joint(*bone),
_ => {
(hand, left_data, right_data) => {
info!("{:?},{:?},{:?}", hand, left_data, right_data);
*status = BoneTrackingStatus::Emulated;
return;
}
};
info!("hand_tracking bone after filter");
if *status == BoneTrackingStatus::Emulated {
*status = BoneTrackingStatus::Tracked;
}

93
src/xr_input/mod.rs
View File

@@ -10,7 +10,7 @@ pub mod trackers;
pub mod xr_camera;
use crate::resources::{XrInstance, XrSession};
use crate::xr_init::{xr_only, XrPostSetup, XrPreSetup, XrSetup};
use crate::xr_init::{xr_only, XrCleanup, XrPostSetup, XrPreSetup, XrSetup};
use crate::xr_input::controllers::XrControllerType;
use crate::xr_input::oculus_touch::setup_oculus_controller;
use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle};
@@ -19,6 +19,7 @@ use bevy::app::{App, PostUpdate, Startup};
use bevy::ecs::entity::Entity;
use bevy::ecs::query::With;
use bevy::ecs::system::Query;
use bevy::hierarchy::DespawnRecursiveExt;
use bevy::log::{info, warn};
use bevy::math::Vec2;
use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource};
@@ -36,7 +37,7 @@ use self::trackers::{
adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye,
OpenXRTrackingRoot,
};
use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCameraType};
use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCamera};
#[derive(Copy, Clone)]
pub struct OpenXrInput {
@@ -56,9 +57,8 @@ impl OpenXrInput {
impl Plugin for OpenXrInput {
fn build(&self, app: &mut App) {
app.add_plugins(CameraProjectionPlugin::<XRProjection>::default());
app.add_plugins(OpenXrActionsPlugin);
app.add_systems(XrPostSetup, post_action_setup_oculus_controller);
// why only when the controller is oculus? that is still backed by generic actions
match self.controller_type {
XrControllerType::OculusTouch => {
app.add_systems(XrSetup, setup_oculus_controller);
@@ -66,77 +66,42 @@ impl Plugin for OpenXrInput {
}
//adopt any new trackers
app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only()));
app.add_systems(PreUpdate, action_set_system.run_if(xr_only()));
app.add_systems(
PreUpdate,
xr_camera_head_sync
.run_if(xr_only())
.after(xr_wait_frame)
.after(locate_views),
);
// app.add_systems(PreUpdate, action_set_system.run_if(xr_only()));
//update controller trackers
app.add_systems(Update, update_open_xr_controllers.run_if(xr_only()));
app.add_systems(
PostUpdate,
update_frusta::<XRProjection>
.after(TransformSystem::TransformPropagate)
.before(VisibilitySystems::UpdatePerspectiveFrusta),
);
app.add_systems(XrPreSetup, init_subaction_path);
app.add_systems(XrSetup, setup_xr_cameras);
app.add_plugins(ExtractComponentPlugin::<XrCameraType>::default());
app.add_plugins(ExtractComponentPlugin::<XRProjection>::default());
app.add_plugins(ExtractComponentPlugin::<TransformExtract>::default());
app.add_plugins(ExtractComponentPlugin::<GlobalTransformExtract>::default());
app.add_systems(XrSetup, setup_xr_root);
app.add_systems(XrCleanup, cleanup_xr_root);
}
}
#[derive(Deref, DerefMut, Resource)]
pub struct InteractionProfileBindings(pub HashMap<&'static str, Vec<Binding<'static>>>);
fn setup_binding_recommendations(
mut commands: Commands,
instance: Res<XrInstance>,
bindings: Res<InteractionProfileBindings>,
) {
commands.remove_resource::<InteractionProfileBindings>();
}
fn setup_xr_cameras(
fn cleanup_xr_root(
mut commands: Commands,
tracking_root_query: Query<Entity, With<OpenXRTrackingRoot>>,
) {
//this needs to do the whole xr tracking volume not just cameras
//get the root?
let tracking_root = match tracking_root_query.get_single() {
Ok(e) => e,
Err(_) => commands
.spawn((SpatialBundle::default(), OpenXRTrackingRoot))
.id(),
};
let right = commands
.spawn((XrCameraBundle::new(Eye::Right), OpenXRRightEye))
.id();
let left = commands
.spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye))
.id();
commands.entity(tracking_root).push_children(&[right, left]);
}
pub fn action_set_system(action_sets: Res<ActionSets>, session: Res<XrSession>) {
let mut active_action_sets = vec![];
for i in &action_sets.0 {
active_action_sets.push(openxr::ActiveActionSet::new(i));
}
//info!("action sets: {:#?}", action_sets.0.len());
match session.sync_actions(&active_action_sets) {
Err(err) => {
warn!("{}", err);
}
_ => {}
for e in &tracking_root_query {
commands.entity(e).despawn_recursive();
}
}
fn setup_xr_root(
mut commands: Commands,
tracking_root_query: Query<Entity, With<OpenXRTrackingRoot>>,
) {
if tracking_root_query.get_single().is_err() {
commands.spawn((SpatialBundle::default(), OpenXRTrackingRoot));
}
}
// pub fn action_set_system(action_sets: Res<ActionSets>, session: Res<XrSession>) {
// let mut active_action_sets = vec![];
// for i in &action_sets.0 {
// active_action_sets.push(openxr::ActiveActionSet::new(i));
// }
// //info!("action sets: {:#?}", action_sets.0.len());
// if let Err(err) = session.sync_actions(&active_action_sets) {
// warn!("{}", err);
// }
// }
pub trait Vec2Conv {
fn to_vec2(&self) -> Vec2;

105
src/xr_input/xr_camera.rs
View File

@@ -1,15 +1,73 @@
use crate::xr_init::{xr_only, XrCleanup, XrSetup};
use crate::xr_input::{QuatConv, Vec3Conv};
use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
use crate::{locate_views, xr_wait_frame, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping};
use bevy::ecs::system::lifetimeless::Read;
use bevy::math::Vec3A;
use bevy::prelude::*;
use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget};
use bevy::render::extract_component::ExtractComponent;
use bevy::render::camera::{
CameraProjection, CameraProjectionPlugin, CameraRenderGraph, RenderTarget,
};
use bevy::render::extract_component::{ExtractComponent, ExtractComponentPlugin};
use bevy::render::primitives::Frustum;
use bevy::render::view::{ColorGrading, ExtractedView, VisibleEntities};
use bevy::render::view::{
update_frusta, ColorGrading, ExtractedView, VisibilitySystems, VisibleEntities,
};
use bevy::transform::TransformSystem;
use openxr::Fovf;
use super::trackers::{OpenXRLeftEye, OpenXRRightEye, OpenXRTracker, OpenXRTrackingRoot};
pub struct XrCameraPlugin;
impl Plugin for XrCameraPlugin {
fn build(&self, app: &mut App) {
app.add_plugins(CameraProjectionPlugin::<XRProjection>::default());
app.add_systems(
PreUpdate,
xr_camera_head_sync
.run_if(xr_only())
.after(xr_wait_frame)
.after(locate_views),
);
// a little late latching
app.add_systems(
PostUpdate,
xr_camera_head_sync
.before(TransformSystem::TransformPropagate)
.run_if(xr_only()),
);
app.add_systems(
PostUpdate,
update_frusta::<XRProjection>
.after(TransformSystem::TransformPropagate)
.before(VisibilitySystems::UpdatePerspectiveFrusta),
);
app.add_systems(XrSetup, setup_xr_cameras);
app.add_systems(XrCleanup, cleanup_xr_cameras);
app.add_plugins(ExtractComponentPlugin::<XrCamera>::default());
app.add_plugins(ExtractComponentPlugin::<XRProjection>::default());
// app.add_plugins(ExtractComponentPlugin::<TransformExtract>::default());
// app.add_plugins(ExtractComponentPlugin::<GlobalTransformExtract>::default());
}
}
// might be unnesesary since it should be parented to the root
fn cleanup_xr_cameras(mut commands: Commands, entities: Query<Entity, With<XrCamera>>) {
for e in &entities {
commands.entity(e).despawn_recursive();
}
}
fn setup_xr_cameras(mut commands: Commands) {
commands.spawn((
XrCameraBundle::new(Eye::Right),
OpenXRRightEye,
OpenXRTracker,
));
commands.spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye, OpenXRTracker));
}
#[derive(Bundle)]
pub struct XrCamerasBundle {
pub left: XrCameraBundle,
@@ -42,13 +100,10 @@ pub struct XrCameraBundle {
pub tonemapping: Tonemapping,
pub dither: DebandDither,
pub color_grading: ColorGrading,
pub xr_camera_type: XrCameraType,
pub xr_camera_type: XrCamera,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)]
pub enum XrCameraType {
Xr(Eye),
Flatscreen,
}
pub struct XrCamera(Eye);
#[derive(Component)]
pub(super) struct TransformExtract;
@@ -108,7 +163,7 @@ impl XrCameraBundle {
tonemapping: Default::default(),
dither: DebandDither::Enabled,
color_grading: Default::default(),
xr_camera_type: XrCameraType::Xr(eye),
xr_camera_type: XrCamera(eye),
}
}
}
@@ -273,15 +328,11 @@ impl CameraProjection for XRProjection {
pub fn xr_camera_head_sync(
views: Res<crate::resources::XrViews>,
mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>,
mut query: Query<(&mut Transform, &XrCamera, &mut XRProjection)>,
) {
//TODO calculate HMD position
for (mut transform, camera_type, mut xr_projection) in query.iter_mut() {
let view_idx = match camera_type {
XrCameraType::Xr(eye) => *eye as usize,
// I don't belive we need a flatscrenn cam, that's just a cam without this component
XrCameraType::Flatscreen => continue,
};
let view_idx = camera_type.0 as usize;
let view = match views.get(view_idx) {
Some(views) => views,
None => continue,
@@ -291,25 +342,3 @@ pub fn xr_camera_head_sync(
transform.translation = view.pose.position.to_vec3();
}
}
pub fn xr_camera_head_sync_render(
views: Res<crate::resources::XrViews>,
mut query: Query<(&mut ExtractedView, &XrCameraType)>,
) {
//TODO calculate HMD position
for (mut transform, camera_type) in query.iter_mut() {
// let mut t = Transform::IDENTITY;
// let view_idx = match camera_type {
// XrCameraType::Xr(eye) => *eye as usize,
// // I don't belive we need a flatscrenn cam, that's just a cam without this component
// XrCameraType::Flatscreen => continue,
// };
// let view = match views.get(view_idx) {
// Some(views) => views,
// None => continue,
// };
// t.rotation = view.pose.orientation.to_quat();
// t.translation = view.pose.position.to_vec3();
info!("cam update");
transform.transform = GlobalTransform::IDENTITY;
}
}