From 31d7b05b4a288aea3e3abddfe476b14f9c02d85f Mon Sep 17 00:00:00 2001 From: Schmarni Date: Thu, 15 Feb 2024 06:04:16 +0100 Subject: [PATCH] handtracking not working on quest 2 v62 --- examples/android/src/lib.rs | 9 ++- src/lib.rs | 17 +++-- src/resources.rs | 1 - src/xr_init/mod.rs | 4 +- src/xr_input/actions.rs | 41 +++++++++-- src/xr_input/hands/common.rs | 2 +- src/xr_input/hands/hand_tracking.rs | 12 +++- src/xr_input/mod.rs | 93 ++++++++---------------- src/xr_input/xr_camera.rs | 105 ++++++++++++++++++---------- 9 files changed, 163 insertions(+), 121 deletions(-) diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index ba0f411..a52a93c 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -6,6 +6,7 @@ use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::passthrough::{PausePassthrough, ResumePassthrough, XrPassthroughState}; use bevy_oxr::xr_init::xr_only; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; +use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer; use bevy_oxr::xr_input::hands::HandBone; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ @@ -17,6 +18,7 @@ use bevy_oxr::DefaultXrPlugins; fn main() { let mut xr_extensions = XrExtensions::default(); xr_extensions.enable_fb_passthrough(); + xr_extensions.enable_hand_tracking(); App::new() .add_plugins(DefaultXrPlugins { reqeusted_extensions: xr_extensions, @@ -28,8 +30,13 @@ fn main() { // .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) + .add_plugins(HandInputDebugRenderer) + .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup) .add_systems(Startup, setup) - .add_systems(Update, (proto_locomotion, toggle_passthrough).run_if(xr_only())) + .add_systems( + Update, + (proto_locomotion, toggle_passthrough).run_if(xr_only()), + ) .add_systems(Update, debug_hand_render.run_if(xr_only())) .add_systems(Startup, spawn_controllers_example) .insert_resource(PrototypeLocomotionConfig::default()) diff --git a/src/lib.rs b/src/lib.rs index 17c5e6c..49c8508 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,12 +6,8 @@ pub mod resources; pub mod xr_init; pub mod xr_input; -use std::fs::File; -use std::io::{BufWriter, Write}; use std::sync::atomic::AtomicBool; -use std::sync::{Arc, Mutex}; -use crate::passthrough::ResumePassthrough; use crate::xr_init::{StartXrSession, XrInitPlugin}; use crate::xr_input::oculus_touch::ActionSets; use bevy::app::{AppExit, PluginGroupBuilder}; @@ -34,10 +30,12 @@ use xr_init::{ xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, SetupXrData, XrEarlyInitPlugin, XrHasWaited, XrShouldRender, XrStatus, }; +use xr_input::actions::OpenXrActionsPlugin; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::HandTrackingPlugin; use xr_input::hands::HandPlugin; +use xr_input::xr_camera::XrCameraPlugin; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -81,7 +79,7 @@ impl Plugin for OpenXrPlugin { debug!("Configured wgpu adapter Features: {:#?}", device.features()); warn!("Starting with OpenXR Instance"); app.insert_resource(xr_instance.clone()); - app.insert_resource(blend_mode.clone()); + app.insert_resource(blend_mode); app.insert_resource(ActionSets(vec![])); app.insert_resource(xr_instance); app.insert_resource(blend_mode); @@ -120,7 +118,9 @@ impl Plugin for OpenXrPlugin { ( xr_reset_per_frame_resources, xr_wait_frame.run_if(xr_only()), + // xr_begin_frame.run_if(xr_only()), locate_views.run_if(xr_only()), + apply_deferred, ) .chain() @@ -215,8 +215,10 @@ impl PluginGroup for DefaultXrPlugins { reqeusted_extensions: self.reqeusted_extensions, app_info: self.app_info.clone(), }) - .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) - .add_after::(XrInitPlugin) + .add(XrInitPlugin) + .add(OpenXrInput::new(XrControllerType::OculusTouch)) + .add(OpenXrActionsPlugin) + .add(XrCameraPlugin) .add_before::(XrEarlyInitPlugin) .add(HandPlugin) .add(HandTrackingPlugin) @@ -318,6 +320,7 @@ pub fn xr_wait_frame( return; } }; + // frame_state.predicted_display_time = xr::Time::from_nanos(frame_state.predicted_display_time.as_nanos() + frame_state.predicted_display_period.as_nanos()); **should_render = frame_state.should_render; **waited = true; } diff --git a/src/resources.rs b/src/resources.rs index 3b6ee26..305c76a 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -16,7 +16,6 @@ use core::ptr; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); -// xr_resource_wrapper!(XrSession, xr::Session); xr_resource_wrapper_copy!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); xr_resource_wrapper_copy!(XrResolution, UVec2); xr_resource_wrapper_copy!(XrFormat, wgpu::TextureFormat); diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs index bd17a04..70ac9c9 100644 --- a/src/xr_init/mod.rs +++ b/src/xr_init/mod.rs @@ -54,6 +54,7 @@ pub fn xr_after_wait_only() -> impl FnMut(Res) -> bool { impl Plugin for XrEarlyInitPlugin { fn build(&self, app: &mut App) { + add_schedules(app); app.add_event::() .add_event::() .add_event::() @@ -63,7 +64,6 @@ impl Plugin for XrEarlyInitPlugin { impl Plugin for XrInitPlugin { fn build(&self, app: &mut App) { - add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); @@ -106,7 +106,9 @@ fn setup_manual_texture_views( } pub fn setup_xr(world: &mut World) { + info!("Pre XrPreSetup"); world.run_schedule(XrPreSetup); + info!("Post XrPreSetup"); world.run_schedule(XrSetup); world.run_schedule(XrPrePostSetup); world.run_schedule(XrPostSetup); diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index b9d8c2d..940f40a 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -6,7 +6,7 @@ use xr::{Action, Binding, Haptic, Posef, Vector2f}; use crate::{ resources::{XrInstance, XrSession}, - xr_init::{XrCleanup, XrPrePostSetup, XrPreSetup}, + xr_init::{xr_only, XrCleanup, XrPrePostSetup, XrPreSetup}, }; use super::oculus_touch::ActionSets; @@ -16,13 +16,18 @@ pub use xr::sys::NULL_PATH; pub struct OpenXrActionsPlugin; impl Plugin for OpenXrActionsPlugin { fn build(&self, app: &mut App) { - app.add_systems(XrPreSetup, insert_setup_action_sets); + app.add_systems(PreUpdate, sync_actions.run_if(xr_only())); + app.add_systems( + XrPreSetup, + (insert_setup_action_sets, apply_deferred).chain(), + ); app.add_systems(XrPrePostSetup, setup_oxr_actions); app.add_systems(XrCleanup, clean_actions); } } fn insert_setup_action_sets(mut cmds: Commands) { + info!("WHAT?!"); cmds.insert_resource(SetupActionSets { sets: HashMap::new(), }); @@ -57,7 +62,6 @@ pub fn setup_oxr_actions(world: &mut World) { let right_path = instance.string_to_path("/user/hand/right").unwrap(); let hands = [left_path, right_path]; - let mut oxr_action_sets = Vec::new(); let mut action_sets = XrActionSets { sets: default() }; // let mut action_bindings: HashMap<&'static str, Vec> = HashMap::new(); let mut action_bindings: HashMap< @@ -101,11 +105,11 @@ pub fn setup_oxr_actions(world: &mut World) { } } } - oxr_action_sets.push(oxr_action_set); + // oxr_action_sets.push(oxr_action_set); action_sets.sets.insert( set_name, ActionSet { - // oxr_action_set, + oxr_action_set, actions, enabled: true, }, @@ -152,10 +156,15 @@ pub fn setup_oxr_actions(world: &mut World) { .expect("Unable to suggest interaction bindings!"); } session - .attach_action_sets(&oxr_action_sets.iter().collect::>()) + .attach_action_sets( + &action_sets + .sets + .values() + .map(|set| &set.oxr_action_set) + .collect::>(), + ) .expect("Unable to attach action sets!"); - world.insert_resource(ActionSets(oxr_action_sets)); world.insert_resource(action_sets); } @@ -267,6 +276,7 @@ pub struct ActionSet { // add functionality to enable/disable action sets enabled: bool, actions: HashMap<&'static str, TypedAction>, + oxr_action_set: xr::ActionSet, } #[derive(Resource)] @@ -380,3 +390,20 @@ impl XrActionSets { } } } + +pub fn sync_actions(action_sets: Res, session: Res) { + let active_sets = action_sets + .sets + .values() + .filter_map(|set| { + if set.enabled { + Some(xr::ActiveActionSet::new(&set.oxr_action_set)) + } else { + None + } + }) + .collect::>(); + if let Err(err) = session.sync_actions(&active_sets) { + warn!("OpenXR action sync error: {}", err); + } +} diff --git a/src/xr_input/hands/common.rs b/src/xr_input/hands/common.rs index f080613..7958e10 100644 --- a/src/xr_input/hands/common.rs +++ b/src/xr_input/hands/common.rs @@ -170,7 +170,7 @@ pub fn spawn_hand_entities(mut commands: Commands) { *bone, OpenXRTracker, *hand, - BoneTrackingStatus::Emulated, + BoneTrackingStatus::Tracked, HandBoneRadius(0.1), )) .id(); diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index e2326a1..0c632f8 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -63,6 +63,7 @@ pub struct HandJoint { pub radius: f32, } +#[derive(Debug)] pub struct HandJoints { inner: [HandJoint; 26], } @@ -175,9 +176,15 @@ pub fn update_hand_bones( }; let left_hand_data = hand_ref.get_poses(Hand::Left); let right_hand_data = hand_ref.get_poses(Hand::Right); + if left_hand_data.is_none() || right_hand_data.is_none() { + error!("something is very wrong for hand_tracking!! doesn't have data for both hands!"); + } + + info!("hand_tracking"); bones .par_iter_mut() .for_each(|(mut transform, hand, bone, mut radius, mut status)| { + info!("hand_tracking bone before filter"); match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) { (Hand::Left, Some(DisableHandTracking::OnlyLeft)) => { *status = BoneTrackingStatus::Emulated; @@ -189,14 +196,17 @@ pub fn update_hand_bones( } _ => {} } + info!("hand_tracking bone mid filter"); let bone_data = match (hand, &left_hand_data, &right_hand_data) { (Hand::Left, Some(data), _) => data.get_joint(*bone), (Hand::Right, _, Some(data)) => data.get_joint(*bone), - _ => { + (hand, left_data, right_data) => { + info!("{:?},{:?},{:?}", hand, left_data, right_data); *status = BoneTrackingStatus::Emulated; return; } }; + info!("hand_tracking bone after filter"); if *status == BoneTrackingStatus::Emulated { *status = BoneTrackingStatus::Tracked; } diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index ee88e4d..13c96df 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -10,7 +10,7 @@ pub mod trackers; pub mod xr_camera; use crate::resources::{XrInstance, XrSession}; -use crate::xr_init::{xr_only, XrPostSetup, XrPreSetup, XrSetup}; +use crate::xr_init::{xr_only, XrCleanup, XrPostSetup, XrPreSetup, XrSetup}; use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; @@ -19,6 +19,7 @@ use bevy::app::{App, PostUpdate, Startup}; use bevy::ecs::entity::Entity; use bevy::ecs::query::With; use bevy::ecs::system::Query; +use bevy::hierarchy::DespawnRecursiveExt; use bevy::log::{info, warn}; use bevy::math::Vec2; use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource}; @@ -36,7 +37,7 @@ use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, }; -use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCameraType}; +use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCamera}; #[derive(Copy, Clone)] pub struct OpenXrInput { @@ -56,9 +57,8 @@ impl OpenXrInput { impl Plugin for OpenXrInput { fn build(&self, app: &mut App) { - app.add_plugins(CameraProjectionPlugin::::default()); - app.add_plugins(OpenXrActionsPlugin); app.add_systems(XrPostSetup, post_action_setup_oculus_controller); + // why only when the controller is oculus? that is still backed by generic actions match self.controller_type { XrControllerType::OculusTouch => { app.add_systems(XrSetup, setup_oculus_controller); @@ -66,77 +66,42 @@ impl Plugin for OpenXrInput { } //adopt any new trackers app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only())); - app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); - app.add_systems( - PreUpdate, - xr_camera_head_sync - .run_if(xr_only()) - .after(xr_wait_frame) - .after(locate_views), - ); + // app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); //update controller trackers app.add_systems(Update, update_open_xr_controllers.run_if(xr_only())); - app.add_systems( - PostUpdate, - update_frusta:: - .after(TransformSystem::TransformPropagate) - .before(VisibilitySystems::UpdatePerspectiveFrusta), - ); app.add_systems(XrPreSetup, init_subaction_path); - app.add_systems(XrSetup, setup_xr_cameras); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); + app.add_systems(XrSetup, setup_xr_root); + app.add_systems(XrCleanup, cleanup_xr_root); } } -#[derive(Deref, DerefMut, Resource)] -pub struct InteractionProfileBindings(pub HashMap<&'static str, Vec>>); - -fn setup_binding_recommendations( - mut commands: Commands, - instance: Res, - bindings: Res, -) { - commands.remove_resource::(); -} - -fn setup_xr_cameras( +fn cleanup_xr_root( mut commands: Commands, tracking_root_query: Query>, ) { - //this needs to do the whole xr tracking volume not just cameras - //get the root? - - let tracking_root = match tracking_root_query.get_single() { - Ok(e) => e, - Err(_) => commands - .spawn((SpatialBundle::default(), OpenXRTrackingRoot)) - .id(), - }; - let right = commands - .spawn((XrCameraBundle::new(Eye::Right), OpenXRRightEye)) - .id(); - let left = commands - .spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye)) - .id(); - commands.entity(tracking_root).push_children(&[right, left]); -} - -pub fn action_set_system(action_sets: Res, session: Res) { - let mut active_action_sets = vec![]; - for i in &action_sets.0 { - active_action_sets.push(openxr::ActiveActionSet::new(i)); - } - //info!("action sets: {:#?}", action_sets.0.len()); - match session.sync_actions(&active_action_sets) { - Err(err) => { - warn!("{}", err); - } - _ => {} + for e in &tracking_root_query { + commands.entity(e).despawn_recursive(); } } +fn setup_xr_root( + mut commands: Commands, + tracking_root_query: Query>, +) { + if tracking_root_query.get_single().is_err() { + commands.spawn((SpatialBundle::default(), OpenXRTrackingRoot)); + } +} + +// pub fn action_set_system(action_sets: Res, session: Res) { +// let mut active_action_sets = vec![]; +// for i in &action_sets.0 { +// active_action_sets.push(openxr::ActiveActionSet::new(i)); +// } +// //info!("action sets: {:#?}", action_sets.0.len()); +// if let Err(err) = session.sync_actions(&active_action_sets) { +// warn!("{}", err); +// } +// } pub trait Vec2Conv { fn to_vec2(&self) -> Vec2; diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 50d38ed..f154d4c 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -1,15 +1,73 @@ +use crate::xr_init::{xr_only, XrCleanup, XrSetup}; use crate::xr_input::{QuatConv, Vec3Conv}; -use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; +use crate::{locate_views, xr_wait_frame, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; use bevy::ecs::system::lifetimeless::Read; use bevy::math::Vec3A; use bevy::prelude::*; -use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; -use bevy::render::extract_component::ExtractComponent; +use bevy::render::camera::{ + CameraProjection, CameraProjectionPlugin, CameraRenderGraph, RenderTarget, +}; +use bevy::render::extract_component::{ExtractComponent, ExtractComponentPlugin}; use bevy::render::primitives::Frustum; -use bevy::render::view::{ColorGrading, ExtractedView, VisibleEntities}; +use bevy::render::view::{ + update_frusta, ColorGrading, ExtractedView, VisibilitySystems, VisibleEntities, +}; +use bevy::transform::TransformSystem; use openxr::Fovf; +use super::trackers::{OpenXRLeftEye, OpenXRRightEye, OpenXRTracker, OpenXRTrackingRoot}; + +pub struct XrCameraPlugin; + +impl Plugin for XrCameraPlugin { + fn build(&self, app: &mut App) { + app.add_plugins(CameraProjectionPlugin::::default()); + app.add_systems( + PreUpdate, + xr_camera_head_sync + .run_if(xr_only()) + .after(xr_wait_frame) + .after(locate_views), + ); + // a little late latching + app.add_systems( + PostUpdate, + xr_camera_head_sync + .before(TransformSystem::TransformPropagate) + .run_if(xr_only()), + ); + app.add_systems( + PostUpdate, + update_frusta:: + .after(TransformSystem::TransformPropagate) + .before(VisibilitySystems::UpdatePerspectiveFrusta), + ); + app.add_systems(XrSetup, setup_xr_cameras); + app.add_systems(XrCleanup, cleanup_xr_cameras); + app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); + // app.add_plugins(ExtractComponentPlugin::::default()); + // app.add_plugins(ExtractComponentPlugin::::default()); + } +} + +// might be unnesesary since it should be parented to the root +fn cleanup_xr_cameras(mut commands: Commands, entities: Query>) { + for e in &entities { + commands.entity(e).despawn_recursive(); + } +} + +fn setup_xr_cameras(mut commands: Commands) { + commands.spawn(( + XrCameraBundle::new(Eye::Right), + OpenXRRightEye, + OpenXRTracker, + )); + commands.spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye, OpenXRTracker)); +} + #[derive(Bundle)] pub struct XrCamerasBundle { pub left: XrCameraBundle, @@ -42,13 +100,10 @@ pub struct XrCameraBundle { pub tonemapping: Tonemapping, pub dither: DebandDither, pub color_grading: ColorGrading, - pub xr_camera_type: XrCameraType, + pub xr_camera_type: XrCamera, } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)] -pub enum XrCameraType { - Xr(Eye), - Flatscreen, -} +pub struct XrCamera(Eye); #[derive(Component)] pub(super) struct TransformExtract; @@ -108,7 +163,7 @@ impl XrCameraBundle { tonemapping: Default::default(), dither: DebandDither::Enabled, color_grading: Default::default(), - xr_camera_type: XrCameraType::Xr(eye), + xr_camera_type: XrCamera(eye), } } } @@ -273,15 +328,11 @@ impl CameraProjection for XRProjection { pub fn xr_camera_head_sync( views: Res, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + mut query: Query<(&mut Transform, &XrCamera, &mut XRProjection)>, ) { //TODO calculate HMD position for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - XrCameraType::Xr(eye) => *eye as usize, - // I don't belive we need a flatscrenn cam, that's just a cam without this component - XrCameraType::Flatscreen => continue, - }; + let view_idx = camera_type.0 as usize; let view = match views.get(view_idx) { Some(views) => views, None => continue, @@ -291,25 +342,3 @@ pub fn xr_camera_head_sync( transform.translation = view.pose.position.to_vec3(); } } -pub fn xr_camera_head_sync_render( - views: Res, - mut query: Query<(&mut ExtractedView, &XrCameraType)>, -) { - //TODO calculate HMD position - for (mut transform, camera_type) in query.iter_mut() { - // let mut t = Transform::IDENTITY; - // let view_idx = match camera_type { - // XrCameraType::Xr(eye) => *eye as usize, - // // I don't belive we need a flatscrenn cam, that's just a cam without this component - // XrCameraType::Flatscreen => continue, - // }; - // let view = match views.get(view_idx) { - // Some(views) => views, - // None => continue, - // }; - // t.rotation = view.pose.orientation.to_quat(); - // t.translation = view.pose.position.to_vec3(); - info!("cam update"); - transform.transform = GlobalTransform::IDENTITY; - } -}