From b59e5be083b3f39a069030877d46789aa66b29a5 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 11 Dec 2023 19:22:21 +0100 Subject: [PATCH 01/80] change type of pretty_name for ActionSets and Actions --- src/xr_input/actions.rs | 26 ++++++++++----------- src/xr_input/hands/emulated.rs | 28 +++++++++++------------ src/xr_input/oculus_touch.rs | 42 +++++++++++++++++----------------- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 827464a..f6d2a59 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -35,40 +35,40 @@ pub fn setup_oxr_actions(world: &mut World) { while let Some((set_name, set)) = a_iter.next() { let mut actions: HashMap<&'static str, TypedAction> = default(); let oxr_action_set = instance - .create_action_set(set_name, set.pretty_name, set.priority) + .create_action_set(set_name, &set.pretty_name, set.priority) .expect("Unable to create action set"); for (action_name, action) in set.actions.into_iter() { let typed_action = match action.action_type { ActionType::F32 => TypedAction::F32(match action.handednes { ActionHandednes::Single => oxr_action_set - .create_action(action_name, action.pretty_name, &[]) + .create_action(action_name, &action.pretty_name, &[]) .expect(&format!("Unable to create action: {}", action_name)), ActionHandednes::Double => oxr_action_set - .create_action(action_name, action.pretty_name, &hands) + .create_action(action_name, &action.pretty_name, &hands) .expect(&format!("Unable to create action: {}", action_name)), }), ActionType::Bool => TypedAction::Bool(match action.handednes { ActionHandednes::Single => oxr_action_set - .create_action(action_name, action.pretty_name, &[]) + .create_action(action_name, &action.pretty_name, &[]) .expect(&format!("Unable to create action: {}", action_name)), ActionHandednes::Double => oxr_action_set - .create_action(action_name, action.pretty_name, &hands) + .create_action(action_name, &action.pretty_name, &hands) .expect(&format!("Unable to create action: {}", action_name)), }), ActionType::PoseF => TypedAction::PoseF(match action.handednes { ActionHandednes::Single => oxr_action_set - .create_action(action_name, action.pretty_name, &[]) + .create_action(action_name, &action.pretty_name, &[]) .expect(&format!("Unable to create action: {}", action_name)), ActionHandednes::Double => oxr_action_set - .create_action(action_name, action.pretty_name, &hands) + .create_action(action_name, &action.pretty_name, &hands) .expect(&format!("Unable to create action: {}", action_name)), }), ActionType::Haptic => TypedAction::Haptic(match action.handednes { ActionHandednes::Single => oxr_action_set - .create_action(action_name, action.pretty_name, &[]) + .create_action(action_name, &action.pretty_name, &[]) .expect(&format!("Unable to create action: {}", action_name)), ActionHandednes::Double => oxr_action_set - .create_action(action_name, action.pretty_name, &hands) + .create_action(action_name, &action.pretty_name, &hands) .expect(&format!("Unable to create action: {}", action_name)), }), }; @@ -164,14 +164,14 @@ pub enum TypedAction { } pub struct SetupAction { - pretty_name: &'static str, + pretty_name: String, action_type: ActionType, handednes: ActionHandednes, bindings: HashMap<&'static str, Vec<&'static str>>, } pub struct SetupActionSet { - pretty_name: &'static str, + pretty_name: String, priority: u32, actions: HashMap<&'static str, SetupAction>, } @@ -180,7 +180,7 @@ impl SetupActionSet { pub fn new_action( &mut self, name: &'static str, - pretty_name: &'static str, + pretty_name: String, action_type: ActionType, handednes: ActionHandednes, ) { @@ -230,7 +230,7 @@ impl SetupActionSets { pub fn add_action_set( &mut self, name: &'static str, - pretty_name: &'static str, + pretty_name: String, priority: u32, ) -> &mut SetupActionSet { self.sets.insert( diff --git a/src/xr_input/hands/emulated.rs b/src/xr_input/hands/emulated.rs index a3423d2..ab1e627 100644 --- a/src/xr_input/hands/emulated.rs +++ b/src/xr_input/hands/emulated.rs @@ -5,8 +5,8 @@ use openxr::{ActionTy, HandJoint}; use super::common::{get_bone_gizmo_style, HandBoneRadius}; use crate::{ - xr_init::{xr_only, XrSetup}, resources::{XrInstance, XrSession}, + xr_init::{xr_only, XrSetup}, xr_input::{ actions::{ ActionHandednes, ActionType, SetupActionSet, SetupActionSets, XrActionSets, XrBinding, @@ -28,10 +28,7 @@ pub struct HandEmulationPlugin; impl Plugin for HandEmulationPlugin { fn build(&self, app: &mut App) { - app.add_systems( - Update, - update_hand_skeleton_from_emulated.run_if(xr_only()), - ); + app.add_systems(Update, update_hand_skeleton_from_emulated.run_if(xr_only())); app.add_systems(XrSetup, setup_hand_emulation_action_set); } } @@ -39,54 +36,55 @@ impl Plugin for HandEmulationPlugin { const HAND_ACTION_SET: &str = "hand_pose_approx"; fn setup_hand_emulation_action_set(mut action_sets: ResMut) { - let mut action_set = action_sets.add_action_set(HAND_ACTION_SET, "Hand Pose Approximaiton", 0); + let action_set = + action_sets.add_action_set(HAND_ACTION_SET, "Hand Pose Approximaiton".into(), 0); action_set.new_action( "thumb_touch", - "Thumb Touched", + "Thumb Touched".into(), ActionType::Bool, ActionHandednes::Double, ); action_set.new_action( "thumb_x", - "Thumb X", + "Thumb X".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "thumb_y", - "Thumb Y", + "Thumb Y".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "index_touch", - "Index Finger Touched", + "Index Finger Touched".into(), ActionType::Bool, ActionHandednes::Double, ); action_set.new_action( "index_value", - "Index Finger Pull", + "Index Finger Pull".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "middle_value", - "Middle Finger Pull", + "Middle Finger Pull".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "ring_value", - "Ring Finger Pull", + "Ring Finger Pull".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "little_value", - "Little Finger Pull", + "Little Finger Pull".into(), ActionType::F32, ActionHandednes::Double, ); @@ -126,6 +124,7 @@ fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) { ); } +#[allow(clippy::type_complexity)] pub(crate) fn update_hand_skeleton_from_emulated( session: Res, instance: Res, @@ -546,5 +545,6 @@ fn get_bone_curl_angle(bone: HandJoint, curl: f32) -> f32 { _ => 1.0, }; let curl_angle = -((mul * curl * 80.0) + 5.0); + #[allow(clippy::needless_return)] return curl_angle; } diff --git a/src/xr_input/oculus_touch.rs b/src/xr_input/oculus_touch.rs index abd501f..6d416bb 100644 --- a/src/xr_input/oculus_touch.rs +++ b/src/xr_input/oculus_touch.rs @@ -305,124 +305,124 @@ pub struct OculusController { impl OculusController { pub fn new(mut action_sets: ResMut) -> anyhow::Result { let action_set = - action_sets.add_action_set("oculus_input", "Oculus Touch Controller Input", 0); + action_sets.add_action_set("oculus_input", "Oculus Touch Controller Input".into(), 0); action_set.new_action( "hand_pose", - "Hand Pose", + "Hand Pose".into(), ActionType::PoseF, ActionHandednes::Double, ); action_set.new_action( "pointer_pose", - "Pointer Pose", + "Pointer Pose".into(), ActionType::PoseF, ActionHandednes::Double, ); action_set.new_action( "squeeze", - "Grip Pull", + "Grip Pull".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "trigger", - "Trigger Pull", + "Trigger Pull".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "trigger_touched", - "Trigger Touch", + "Trigger Touch".into(), ActionType::Bool, ActionHandednes::Double, ); action_set.new_action( "haptic_feedback", - "Haptic Feedback", + "Haptic Feedback".into(), ActionType::Haptic, ActionHandednes::Double, ); action_set.new_action( "x_button", - "X Button", + "X Button".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "x_button_touch", - "X Button Touch", + "X Button Touch".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "y_button", - "Y Button", + "Y Button".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "y_button_touch", - "Y Button Touch", + "Y Button Touch".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "a_button", - "A Button", + "A Button".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "a_button_touch", - "A Button Touch", + "A Button Touch".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "b_button", - "B Button", + "B Button".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "b_button_touch", - "B Button Touch", + "B Button Touch".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "menu_button", - "Menu Button", + "Menu Button".into(), ActionType::Bool, ActionHandednes::Single, ); action_set.new_action( "thumbstick_x", - "Thumbstick X", + "Thumbstick X".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "thumbstick_y", - "Thumbstick y", + "Thumbstick y".into(), ActionType::F32, ActionHandednes::Double, ); action_set.new_action( "thumbstick_touch", - "Thumbstick Touch", + "Thumbstick Touch".into(), ActionType::Bool, ActionHandednes::Double, ); action_set.new_action( "thumbstick_click", - "Thumbstick Click", + "Thumbstick Click".into(), ActionType::Bool, ActionHandednes::Double, ); action_set.new_action( "thumbrest_touch", - "Thumbrest Touch", + "Thumbrest Touch".into(), ActionType::Bool, ActionHandednes::Double, ); From ae8c5ef1853ebece0136c50ee957256f83bf3ae0 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 11 Dec 2023 19:24:33 +0100 Subject: [PATCH 02/80] update XrSetup Scheduling to allow other plugins to run before in Startup --- src/xr_init.rs | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/xr_init.rs b/src/xr_init.rs index 777d777..945fd77 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -8,17 +8,14 @@ use bevy::{ prelude::*, render::{ extract_resource::{ExtractResource, ExtractResourcePlugin}, - renderer::{ - self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, - }, + renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}, settings::WgpuSettings, }, - window::{PrimaryWindow, RawHandleWrapper}, + window::RawHandleWrapper, }; use wgpu::Instance; use crate::{ - graphics, input::XrInput, resources::{ XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, @@ -107,9 +104,7 @@ impl Plugin for RenderRestartPlugin { .insert_resource(ForceMain) .add_event::() .add_event::() - .add_systems(PreStartup, xr_presetup.run_if(xr_only())) - .add_systems(Startup, xr_setup.run_if(xr_only())) - .add_systems(PostStartup, xr_postsetup.run_if(xr_only())) + .add_systems(PostStartup, setup_xr.run_if(xr_only())) .add_systems( PostUpdate, update_xr_stuff.run_if(on_event::()), @@ -154,17 +149,6 @@ fn add_schedules(app: &mut App) { } } -fn xr_presetup(world: &mut World) { - world.run_schedule(XrPreSetup); -} -fn xr_setup(world: &mut World) { - world.run_schedule(XrSetup); -} -fn xr_postsetup(world: &mut World) { - world.run_schedule(XrPrePostSetup); - world.run_schedule(XrPostSetup); -} - fn setup_xr(world: &mut World) { world.run_schedule(XrPreSetup); world.run_schedule(XrSetup); From 900d4fe725a4f88b3478bf221875382ee028c5da Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 11 Dec 2023 19:52:40 +0100 Subject: [PATCH 03/80] Make ActionType Copy --- src/xr_input/actions.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index f6d2a59..10a0914 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -149,6 +149,7 @@ pub enum ActionHandednes { Double, } +#[derive(Clone, Copy)] pub enum ActionType { F32, Bool, From 2e492e120ca1977f3b3469b1c2e8392354ab0bb9 Mon Sep 17 00:00:00 2001 From: Schmarni <51007916+Schmarni-Dev@users.noreply.github.com> Date: Mon, 11 Dec 2023 22:07:46 +0100 Subject: [PATCH 04/80] made setup_xr public (#56) * made setup_xr public * made the action syncing system public too --- src/xr_init.rs | 2 +- src/xr_input/mod.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/xr_init.rs b/src/xr_init.rs index 945fd77..499e7ce 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -149,7 +149,7 @@ fn add_schedules(app: &mut App) { } } -fn setup_xr(world: &mut World) { +pub fn setup_xr(world: &mut World) { world.run_schedule(XrPreSetup); world.run_schedule(XrSetup); world.run_schedule(XrPrePostSetup); diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index e49f045..f436ec4 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -9,14 +9,14 @@ pub mod prototype_locomotion; pub mod trackers; pub mod xr_camera; -use crate::xr_init::{XrPostSetup, XrSetup, xr_only}; use crate::resources::{XrInstance, XrSession}; use crate::xr_begin_frame; +use crate::xr_init::{xr_only, XrPostSetup, XrSetup}; use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; use bevy::app::{App, PostUpdate, Startup}; -use bevy::log::{warn, info}; +use bevy::log::{info, warn}; use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource}; use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3}; use bevy::render::camera::CameraProjectionPlugin; @@ -52,10 +52,7 @@ impl Plugin for OpenXrInput { fn build(&self, app: &mut App) { app.add_plugins(CameraProjectionPlugin::::default()); app.add_plugins(OpenXrActionsPlugin); - app.add_systems( - XrPostSetup, - post_action_setup_oculus_controller, - ); + app.add_systems(XrPostSetup, post_action_setup_oculus_controller); match self.controller_type { XrControllerType::OculusTouch => { app.add_systems(XrSetup, setup_oculus_controller); @@ -64,7 +61,10 @@ impl Plugin for OpenXrInput { //adopt any new trackers app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only())); app.add_systems(PreUpdate, action_set_system); - app.add_systems(PreUpdate, xr_camera_head_sync.run_if(xr_only()).after(xr_begin_frame)); + app.add_systems( + PreUpdate, + xr_camera_head_sync.run_if(xr_only()).after(xr_begin_frame), + ); //update controller trackers app.add_systems(Update, update_open_xr_controllers.run_if(xr_only())); app.add_systems( @@ -103,7 +103,7 @@ fn setup_xr_cameras(mut commands: Commands) { commands.entity(tracking_root).push_children(&[right, left]); } -fn action_set_system(action_sets: Res, session: Res) { +pub fn action_set_system(action_sets: Res, session: Res) { let mut active_action_sets = vec![]; for i in &action_sets.0 { active_action_sets.push(openxr::ActiveActionSet::new(i)); From 248d160ae524183b14cdd1e3121ae4914b064771 Mon Sep 17 00:00:00 2001 From: Schmarni <51007916+Schmarni-Dev@users.noreply.github.com> Date: Tue, 12 Dec 2023 01:26:55 +0100 Subject: [PATCH 05/80] add support for Vec2 Actions, slightly changed the way actions are created but no functionality change and changed the get_action_* return result to a custom error (#57) --- src/xr_input/actions.rs | 155 +++++++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 56 deletions(-) diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 10a0914..47802fd 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -1,8 +1,13 @@ +use std::error::Error; + use bevy::{prelude::*, utils::HashMap}; use openxr as xr; -use xr::{Action, Binding, Haptic, Posef}; +use xr::{Action, Binding, Haptic, Posef, Vector2f}; -use crate::{resources::{XrInstance, XrSession}, xr_init::XrPrePostSetup}; +use crate::{ + resources::{XrInstance, XrSession}, + xr_init::XrPrePostSetup, +}; use super::oculus_touch::ActionSets; @@ -16,6 +21,22 @@ impl Plugin for OpenXrActionsPlugin { } } +#[inline(always)] +fn create_action( + action: &SetupAction, + action_name: &'static str, + oxr_action_set: &xr::ActionSet, + hands: &[xr::Path], +) -> xr::Action { + match action.handednes { + ActionHandednes::Single => oxr_action_set + .create_action(action_name, &action.pretty_name, &[]) + .unwrap_or_else(|_| panic!("Unable to create action: {}", action_name)), + ActionHandednes::Double => oxr_action_set + .create_action(action_name, &action.pretty_name, hands) + .unwrap_or_else(|_| panic!("Unable to create action: {}", action_name)), + } +} pub fn setup_oxr_actions(world: &mut World) { let actions = world.remove_resource::().unwrap(); let instance = world.get_resource::().unwrap(); @@ -27,50 +48,33 @@ pub fn setup_oxr_actions(world: &mut World) { let mut oxr_action_sets = Vec::new(); let mut action_sets = XrActionSets { sets: default() }; // let mut action_bindings: HashMap<&'static str, Vec> = HashMap::new(); - let mut a_iter = actions.sets.into_iter(); let mut action_bindings: HashMap< (&'static str, &'static str), HashMap<&'static str, Vec>, > = HashMap::new(); - while let Some((set_name, set)) = a_iter.next() { + for (set_name, set) in actions.sets.into_iter() { let mut actions: HashMap<&'static str, TypedAction> = default(); let oxr_action_set = instance .create_action_set(set_name, &set.pretty_name, set.priority) .expect("Unable to create action set"); for (action_name, action) in set.actions.into_iter() { + use self::create_action as ca; let typed_action = match action.action_type { - ActionType::F32 => TypedAction::F32(match action.handednes { - ActionHandednes::Single => oxr_action_set - .create_action(action_name, &action.pretty_name, &[]) - .expect(&format!("Unable to create action: {}", action_name)), - ActionHandednes::Double => oxr_action_set - .create_action(action_name, &action.pretty_name, &hands) - .expect(&format!("Unable to create action: {}", action_name)), - }), - ActionType::Bool => TypedAction::Bool(match action.handednes { - ActionHandednes::Single => oxr_action_set - .create_action(action_name, &action.pretty_name, &[]) - .expect(&format!("Unable to create action: {}", action_name)), - ActionHandednes::Double => oxr_action_set - .create_action(action_name, &action.pretty_name, &hands) - .expect(&format!("Unable to create action: {}", action_name)), - }), - ActionType::PoseF => TypedAction::PoseF(match action.handednes { - ActionHandednes::Single => oxr_action_set - .create_action(action_name, &action.pretty_name, &[]) - .expect(&format!("Unable to create action: {}", action_name)), - ActionHandednes::Double => oxr_action_set - .create_action(action_name, &action.pretty_name, &hands) - .expect(&format!("Unable to create action: {}", action_name)), - }), - ActionType::Haptic => TypedAction::Haptic(match action.handednes { - ActionHandednes::Single => oxr_action_set - .create_action(action_name, &action.pretty_name, &[]) - .expect(&format!("Unable to create action: {}", action_name)), - ActionHandednes::Double => oxr_action_set - .create_action(action_name, &action.pretty_name, &hands) - .expect(&format!("Unable to create action: {}", action_name)), - }), + ActionType::Vec2 => { + TypedAction::Vec2(ca(&action, action_name, &oxr_action_set, &hands)) + } + ActionType::F32 => { + TypedAction::F32(ca(&action, action_name, &oxr_action_set, &hands)) + } + ActionType::Bool => { + TypedAction::Bool(ca(&action, action_name, &oxr_action_set, &hands)) + } + ActionType::PoseF => { + TypedAction::PoseF(ca(&action, action_name, &oxr_action_set, &hands)) + } + ActionType::Haptic => { + TypedAction::Haptic(ca(&action, action_name, &oxr_action_set, &hands)) + } }; actions.insert(action_name, typed_action); for (device_path, bindings) in action.bindings.into_iter() { @@ -107,18 +111,19 @@ pub fn setup_oxr_actions(world: &mut World) { .zip([&action_bindings].into_iter().cycle()) .flat_map(move |((set_name, action_name, action), bindings)| { bindings - .get(&(set_name.clone(), action_name.clone())) + .get(&(set_name as &'static str, action_name as &'static str)) .unwrap() .iter() - .map(move |(dev, bindings)| (action.clone(), dev.clone(), bindings)) + .map(move |(dev, bindings)| (action, dev, bindings)) }) .map(|(action, dev, bindings)| { info!("Hi"); ( dev, bindings - .into_iter() + .iter() .map(move |binding| match &action { + TypedAction::Vec2(a) => Binding::new(a, *binding), TypedAction::F32(a) => Binding::new(a, *binding), TypedAction::Bool(a) => Binding::new(a, *binding), TypedAction::PoseF(a) => Binding::new(a, *binding), @@ -155,6 +160,7 @@ pub enum ActionType { Bool, PoseF, Haptic, + Vec2, } pub enum TypedAction { @@ -162,6 +168,7 @@ pub enum TypedAction { Bool(Action), PoseF(Action), Haptic(Action), + Vec2(Action), } pub struct SetupAction { @@ -247,7 +254,7 @@ impl SetupActionSets { } pub struct ActionSet { - // oxr_action_set: xr::ActionSet, + // add functionality to enable/disable action sets enabled: bool, actions: HashMap<&'static str, TypedAction>, } @@ -257,73 +264,109 @@ pub struct XrActionSets { sets: HashMap<&'static str, ActionSet>, } +use std::fmt::Display as FmtDisplay; +impl FmtDisplay for ActionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let err = match self { + ActionError::NoActionSet => "Action Set Not Found!", + ActionError::NoAction => "Action Not Found!", + ActionError::WrongActionType => "Wrong Action Type!", + }; + write!(f, "{}", err) + } +} +impl Error for ActionError {} +#[derive(Debug)] +pub enum ActionError { + NoActionSet, + NoAction, + WrongActionType, +} + impl XrActionSets { + pub fn get_action_vec2( + &self, + action_set: &'static str, + action_name: &'static str, + ) -> Result<&Action, ActionError> { + let action = self + .sets + .get(action_set) + .ok_or(ActionError::NoActionSet)? + .actions + .get(action_name) + .ok_or(ActionError::NoAction)?; + match action { + TypedAction::Vec2(a) => Ok(a), + _ => Err(ActionError::WrongActionType), + } + } pub fn get_action_f32( &self, action_set: &'static str, action_name: &'static str, - ) -> anyhow::Result<&Action> { + ) -> Result<&Action,ActionError> { let action = self .sets .get(action_set) - .ok_or(anyhow::anyhow!("Action Set Not Found!"))? + .ok_or(ActionError::NoActionSet)? .actions .get(action_name) - .ok_or(anyhow::anyhow!("Action Not Found!"))?; + .ok_or(ActionError::NoAction)?; match action { TypedAction::F32(a) => Ok(a), - _ => anyhow::bail!("wrong action type"), + _ => Err(ActionError::WrongActionType), } } pub fn get_action_bool( &self, action_set: &'static str, action_name: &'static str, - ) -> anyhow::Result<&Action> { + ) -> Result<&Action,ActionError> { let action = self .sets .get(action_set) - .ok_or(anyhow::anyhow!("Action Set Not Found!"))? + .ok_or(ActionError::NoActionSet)? .actions .get(action_name) - .ok_or(anyhow::anyhow!("Action Not Found!"))?; + .ok_or(ActionError::NoAction)?; match action { TypedAction::Bool(a) => Ok(a), - _ => anyhow::bail!("wrong action type"), + _ => Err(ActionError::WrongActionType), } } pub fn get_action_posef( &self, action_set: &'static str, action_name: &'static str, - ) -> anyhow::Result<&Action> { + ) -> Result<&Action,ActionError> { let action = self .sets .get(action_set) - .ok_or(anyhow::anyhow!("Action Set Not Found!"))? + .ok_or(ActionError::NoActionSet)? .actions .get(action_name) - .ok_or(anyhow::anyhow!("Action Not Found!"))?; + .ok_or(ActionError::NoAction)?; match action { TypedAction::PoseF(a) => Ok(a), - _ => anyhow::bail!("wrong action type"), + _ => Err(ActionError::WrongActionType), } } pub fn get_action_haptic( &self, action_set: &'static str, action_name: &'static str, - ) -> anyhow::Result<&Action> { + ) -> Result<&Action,ActionError> { let action = self .sets .get(action_set) - .ok_or(anyhow::anyhow!("Action Set Not Found!"))? + .ok_or(ActionError::NoActionSet)? .actions .get(action_name) - .ok_or(anyhow::anyhow!("Action Not Found!"))?; + .ok_or(ActionError::NoAction)?; match action { TypedAction::Haptic(a) => Ok(a), - _ => anyhow::bail!("wrong action type"), + _ => Err(ActionError::WrongActionType), } } } From 265696b07da0bbda49bec7a280d3ef07858cd201 Mon Sep 17 00:00:00 2001 From: Schmarni <51007916+Schmarni-Dev@users.noreply.github.com> Date: Thu, 14 Dec 2023 06:25:17 +0100 Subject: [PATCH 06/80] Add Vec2Conv helper and Display warning with message when openxr fails to load (#58) * add support for Vec2 Actions, slightly changed the way actions are created but no functionality change and changed the get_action_* return result to a custom error * Whoops * added warning with error when openxr fails to load and added Vec2Conv helper * made it possible for the user to spawn OpenXRTrackingRoot --- src/lib.rs | 123 +++++++++++++++++++++++--------------------- src/xr_input/mod.rs | 30 +++++++++-- 2 files changed, 88 insertions(+), 65 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 296c6d1..7142773 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,67 +82,70 @@ impl Plugin for OpenXrPlugin { let mut system_state: SystemState>> = SystemState::new(&mut app.world); let primary_window = system_state.get(&app.world).get_single().ok().cloned(); - if let Ok(( - device, - queue, - adapter_info, - render_adapter, - instance, - xr_instance, - session, - blend_mode, - resolution, - format, - session_running, - frame_waiter, - swapchain, - input, - views, - frame_state, - )) = graphics::initialize_xr_graphics(primary_window.clone()) - { - // std::thread::sleep(Duration::from_secs(5)); - debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); - debug!("Configured wgpu adapter Features: {:#?}", device.features()); - app.insert_resource(xr_instance.clone()); - app.insert_resource(session.clone()); - app.insert_resource(blend_mode.clone()); - app.insert_resource(resolution.clone()); - app.insert_resource(format.clone()); - app.insert_resource(session_running.clone()); - app.insert_resource(frame_waiter.clone()); - app.insert_resource(swapchain.clone()); - app.insert_resource(input.clone()); - app.insert_resource(views.clone()); - app.insert_resource(frame_state.clone()); - let xr_data = XrRenderData { + match graphics::initialize_xr_graphics(primary_window.clone()) { + Ok(( + device, + queue, + adapter_info, + render_adapter, + instance, xr_instance, - xr_session: session, - xr_blend_mode: blend_mode, - xr_resolution: resolution, - xr_format: format, - xr_session_running: session_running, - xr_frame_waiter: frame_waiter, - xr_swapchain: swapchain, - xr_input: input, - xr_views: views, - xr_frame_state: frame_state, - }; - app.insert_resource(xr_data); - app.insert_resource(ActionSets(vec![])); - app.add_plugins(RenderPlugin { - render_creation: RenderCreation::Manual( - device, - queue, - adapter_info, - render_adapter, - RenderInstance(Arc::new(instance)), - ), - }); - app.insert_resource(XrEnableStatus::Enabled); - } else { - app.add_plugins(RenderPlugin::default()); - app.insert_resource(XrEnableStatus::Disabled); + session, + blend_mode, + resolution, + format, + session_running, + frame_waiter, + swapchain, + input, + views, + frame_state, + )) => { + // std::thread::sleep(Duration::from_secs(5)); + debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); + debug!("Configured wgpu adapter Features: {:#?}", device.features()); + app.insert_resource(xr_instance.clone()); + app.insert_resource(session.clone()); + app.insert_resource(blend_mode.clone()); + app.insert_resource(resolution.clone()); + app.insert_resource(format.clone()); + app.insert_resource(session_running.clone()); + app.insert_resource(frame_waiter.clone()); + app.insert_resource(swapchain.clone()); + app.insert_resource(input.clone()); + app.insert_resource(views.clone()); + app.insert_resource(frame_state.clone()); + let xr_data = XrRenderData { + xr_instance, + xr_session: session, + xr_blend_mode: blend_mode, + xr_resolution: resolution, + xr_format: format, + xr_session_running: session_running, + xr_frame_waiter: frame_waiter, + xr_swapchain: swapchain, + xr_input: input, + xr_views: views, + xr_frame_state: frame_state, + }; + app.insert_resource(xr_data); + app.insert_resource(ActionSets(vec![])); + app.add_plugins(RenderPlugin { + render_creation: RenderCreation::Manual( + device, + queue, + adapter_info, + render_adapter, + RenderInstance(Arc::new(instance)), + ), + }); + app.insert_resource(XrEnableStatus::Enabled); + } + Err(err) => { + warn!("OpenXR Failed to initialize: {}", err); + app.add_plugins(RenderPlugin::default()); + app.insert_resource(XrEnableStatus::Disabled); + } } } diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index f436ec4..50268c8 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -16,7 +16,11 @@ use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; use bevy::app::{App, PostUpdate, Startup}; +use bevy::ecs::entity::Entity; +use bevy::ecs::query::With; +use bevy::ecs::system::Query; use bevy::log::{info, warn}; +use bevy::math::Vec2; use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource}; use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3}; use bevy::render::camera::CameraProjectionPlugin; @@ -60,7 +64,7 @@ impl Plugin for OpenXrInput { } //adopt any new trackers app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only())); - app.add_systems(PreUpdate, action_set_system); + app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); app.add_systems( PreUpdate, xr_camera_head_sync.run_if(xr_only()).after(xr_begin_frame), @@ -88,12 +92,19 @@ fn setup_binding_recommendations( commands.remove_resource::(); } -fn setup_xr_cameras(mut commands: Commands) { +fn setup_xr_cameras( + mut commands: Commands, + tracking_root_query: Query>, +) { //this needs to do the whole xr tracking volume not just cameras //get the root? - let tracking_root = commands - .spawn((SpatialBundle::default(), OpenXRTrackingRoot)) - .id(); + + let tracking_root = match tracking_root_query.get_single() { + Ok(e) => e, + Err(_) => commands + .spawn((SpatialBundle::default(), OpenXRTrackingRoot)) + .id(), + }; let right = commands .spawn((XrCameraBundle::new(Eye::Right), OpenXRRightEye)) .id(); @@ -117,6 +128,15 @@ pub fn action_set_system(action_sets: Res, session: Res) } } +pub trait Vec2Conv { + fn to_vec2(&self) -> Vec2; +} + +impl Vec2Conv for openxr::Vector2f { + fn to_vec2(&self) -> Vec2 { + Vec2::new(self.x, self.y) + } +} pub trait Vec3Conv { fn to_vec3(&self) -> Vec3; } From fcedb9c0c892915f9fb549f5530c698f39aa2283 Mon Sep 17 00:00:00 2001 From: Schmarni <51007916+Schmarni-Dev@users.noreply.github.com> Date: Fri, 15 Dec 2023 07:09:07 +0100 Subject: [PATCH 07/80] do not compile openxrs for wasm32 (#59) --- Cargo.toml | 4 ++-- src/lib.rs | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 57a7a2a..bd61fc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,10 +24,10 @@ wgpu = "0.17.1" wgpu-core = { version = "0.17.1", features = ["vulkan"] } wgpu-hal = "0.17.1" -[target.'cfg( target_family = "unix" )'.dependencies] +[target.'cfg(all(target_family = "unix", not(target_arch = "wasm32")) )'.dependencies] openxr = { version = "0.17.1", features = ["mint"] } -[target.'cfg(not(target_family = "unix"))'.dependencies] +[target.'cfg(all(not(target_family = "unix"), not(target_arch = "wasm32")))'.dependencies] openxr = { version = "0.17.1", features = ["mint", "static"] } [dev-dependencies] diff --git a/src/lib.rs b/src/lib.rs index 7142773..3957e5e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,6 +82,8 @@ impl Plugin for OpenXrPlugin { let mut system_state: SystemState>> = SystemState::new(&mut app.world); let primary_window = system_state.get(&app.world).get_single().ok().cloned(); + + #[cfg(not(target_arch = "wasm32"))] match graphics::initialize_xr_graphics(primary_window.clone()) { Ok(( device, @@ -147,6 +149,11 @@ impl Plugin for OpenXrPlugin { app.insert_resource(XrEnableStatus::Disabled); } } + #[cfg(target_arch = "wasm32")] + { + app.add_plugins(RenderPlugin::default()); + app.insert_resource(XrEnableStatus::Disabled); + } } fn ready(&self, app: &App) -> bool { From 65a30cac3c4e77d79730d6ec369e9377add47cde Mon Sep 17 00:00:00 2001 From: Schmarni <51007916+Schmarni-Dev@users.noreply.github.com> Date: Mon, 1 Jan 2024 00:35:59 +0100 Subject: [PATCH 08/80] make app close when requested by os and other stuff (#61) * idk * mr stuff * done * fix windows * add patches to Readme --- Cargo.toml | 10 +- README.md | 11 ++ examples/android/Cargo.toml | 6 +- examples/android/src/lib.rs | 8 +- examples/demo/Cargo.toml | 19 ++- examples/demo/manifest.yaml | 5 +- examples/demo/src/lib.rs | 21 ++- examples/demo/src/setup.rs | 4 +- examples/globe.rs | 8 +- examples/xr.rs | 8 +- src/graphics/extensions.rs | 242 ++++++++++++++++++++++++++++ src/graphics/mod.rs | 36 ++++- src/graphics/vulkan.rs | 45 ++++-- src/input.rs | 30 ++-- src/lib.rs | 96 +++++++---- src/resources.rs | 50 ++++++ src/xr_input/actions.rs | 12 +- src/xr_input/hands/common.rs | 4 +- src/xr_input/hands/hand_tracking.rs | 17 +- src/xr_input/hands/mod.rs | 2 +- src/xr_input/mod.rs | 5 +- src/xr_input/oculus_touch.rs | 192 +++++++++++++++------- 22 files changed, 666 insertions(+), 165 deletions(-) create mode 100644 src/graphics/extensions.rs diff --git a/Cargo.toml b/Cargo.toml index bd61fc9..c35974e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,8 @@ license = "MIT/Apache-2.0" [features] -default = ["linked"] -linked = ["openxr/linked"] +default = [] +force-link = ["openxr/linked"] [workspace] members = ["examples/android", "examples/demo"] @@ -24,9 +24,10 @@ wgpu = "0.17.1" wgpu-core = { version = "0.17.1", features = ["vulkan"] } wgpu-hal = "0.17.1" +[target.'cfg(windows)'.dependencies] +openxr = { version = "0.17.1", features = ["linked","static","mint"] } [target.'cfg(all(target_family = "unix", not(target_arch = "wasm32")) )'.dependencies] openxr = { version = "0.17.1", features = ["mint"] } - [target.'cfg(all(not(target_family = "unix"), not(target_arch = "wasm32")))'.dependencies] openxr = { version = "0.17.1", features = ["mint", "static"] } @@ -41,3 +42,6 @@ path = "examples/xr.rs" [profile.release] debug = true + +[patch.crates-io] +ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } diff --git a/README.md b/README.md index 8fe5d79..5bdf0b1 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,14 @@ To see it in action run the example in `examples` with `cargo run --example xr` - Make sure, if you're on Linux, that you have the `openxr` package installed on your system. - I'm getting poor performance. - Like other bevy projects, make sure you're building in release (example: `cargo run --example xr --release`) + +## Recommendations + +for now we recommend you to add the folowing to your root Cargo.toml +``` +[patch.crates-io] +ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +ndk-sys = { package = "ndk-sys", git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +ndk-context = { package = "ndk-context", git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +bevy_pbr = { package = "bevy_pbr", git = "https://github.com/MalekiRe/bevy", branch = "release-0.12.1" } +``` diff --git a/examples/android/Cargo.toml b/examples/android/Cargo.toml index 2a52a8a..9f3f65a 100644 --- a/examples/android/Cargo.toml +++ b/examples/android/Cargo.toml @@ -10,13 +10,9 @@ license = "MIT OR Apache-2.0" name = "bevy_openxr_android" crate-type = ["rlib", "cdylib"] -[target.'cfg(not(target_os="android"))'.dependencies.bevy_oxr] -path = "../../" -default-features = true - [dependencies] -bevy_oxr = { path = "../..", default-features = false } +bevy_oxr.path = "../.." bevy = "0.12" openxr = { git = "https://github.com/Ralith/openxrs", features = ["mint"] } diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index 4f8bdeb..4fabb2b 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -1,6 +1,7 @@ use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}; use bevy::prelude::*; use bevy::transform::components::Transform; +use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ @@ -11,7 +12,12 @@ use bevy_oxr::DefaultXrPlugins; #[bevy_main] fn main() { App::new() - .add_plugins(DefaultXrPlugins) + .add_plugins(DefaultXrPlugins { + app_info: XrAppInfo { + name: "Bevy OXR Android Example".into(), + }, + ..default() + }) .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) diff --git a/examples/demo/Cargo.toml b/examples/demo/Cargo.toml index a6abbf9..14e39de 100644 --- a/examples/demo/Cargo.toml +++ b/examples/demo/Cargo.toml @@ -10,12 +10,21 @@ crate-type = ["rlib", "cdylib"] [dependencies] bevy = "0.12" -bevy_oxr = { path = "../../", default-features = false } +bevy_oxr.path = "../../" bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } color-eyre = "0.6.2" -[target.'cfg(not(target_os="android"))'.dependencies.bevy_oxr] -path = "../../" -# May need to be more specific. needs to be false at least on linux without an active runtime to run in flat -default-features = true +# [target.'cfg(not(target_os="android"))'.dependencies.bevy_oxr] +# path = "../../" +# # May need to be more specific. needs to be false at least on linux without an active runtime to run in flat +# default-features = true + +[profile.release] +debug = true + +[patch.crates-io] +ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +ndk-sys = { package = "ndk-sys", git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +ndk-context = { package = "ndk-context", git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +ndk-glue = { package = "ndk-glue", git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } diff --git a/examples/demo/manifest.yaml b/examples/demo/manifest.yaml index ab098d6..730187a 100644 --- a/examples/demo/manifest.yaml +++ b/examples/demo/manifest.yaml @@ -3,13 +3,14 @@ android: - "runtime_libs" manifest: package: "org.bevyengine.demo_openxr_android" + # Are features and permissions fliped? uses_feature: - name: "android.hardware.vr.headtracking" required: true - name: "oculus.software.handtracking" required: false - - name: "com.oculus.experimental.enabled" - required: true + # - name: "com.oculus.feature.PASSTHROUGH" + # required: true uses_permission: - name: "com.oculus.permission.HAND_TRACKING" application: diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index 9ce5000..22529c7 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -16,9 +16,10 @@ use bevy::{ transform::TransformSystem, }; use bevy_oxr::{ + graphics::{extensions::XrExtensions, XrAppInfo, XrPreferdBlendMode}, input::XrInput, - xr_init::{XrEnableRequest, XrEnableStatus, xr_only}, resources::{XrFrameState, XrInstance, XrSession}, + xr_init::{xr_only, XrEnableRequest, XrEnableStatus}, xr_input::{ actions::XrActionSets, debug_gizmos::OpenXrDebugRenderer, @@ -31,7 +32,10 @@ use bevy_oxr::{ }, oculus_touch::OculusController, prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}, - trackers::{OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker}, + trackers::{ + OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker, + OpenXRTrackingRoot, + }, Hand, }, DefaultXrPlugins, @@ -61,13 +65,20 @@ pub fn main() { info!("Running bevy_openxr demo"); let mut app = App::new(); + let mut xr_extensions = XrExtensions::default(); app.add_systems(Update, input_stuff) //lets get the usual diagnostic stuff added .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) //lets get the xr defaults added - .add_plugins(DefaultXrPlugins) + .add_plugins(DefaultXrPlugins { + reqeusted_extensions: xr_extensions, + prefered_blend_mode: XrPreferdBlendMode::Opaque, + app_info: XrAppInfo { + name: "Bevy OXR Demo".into(), + }, + }) //lets add the debug renderer for the controllers .add_plugins(OpenXrDebugRenderer) //rapier goes here @@ -93,7 +104,9 @@ pub fn main() { //draw the interaction gizmos .add_systems( Update, - draw_interaction_gizmos.run_if(xr_only()).after(update_interactable_states), + draw_interaction_gizmos + .run_if(xr_only()) + .after(update_interactable_states), ) .add_systems(Update, draw_socket_gizmos.after(update_interactable_states)) //add our cube spawning system diff --git a/examples/demo/src/setup.rs b/examples/demo/src/setup.rs index 46d4d51..5d3157f 100644 --- a/examples/demo/src/setup.rs +++ b/examples/demo/src/setup.rs @@ -8,7 +8,7 @@ use bevy::{ }; use bevy_oxr::xr_input::interactions::{Touched, XRInteractable, XRInteractableState}; use bevy_rapier3d::{ - prelude::{Collider, RigidBody, Group, CollisionGroups}, + prelude::{Collider, CollisionGroups, Group, RigidBody}, render::ColliderDebugColor, }; @@ -67,7 +67,7 @@ pub fn setup_scene( // }); // camera commands.spawn((Camera3dBundle { - transform: Transform::from_xyz(0.25, 1.25, 0.0).looking_at( + transform: Transform::from_xyz(5.25, 5.25, 5.0).looking_at( Vec3 { x: -0.548, y: -0.161, diff --git a/examples/globe.rs b/examples/globe.rs index fef489a..cf58bea 100644 --- a/examples/globe.rs +++ b/examples/globe.rs @@ -1,6 +1,7 @@ use bevy::diagnostic::LogDiagnosticsPlugin; use bevy::prelude::*; use bevy::transform::components::Transform; +use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::resources::XrViews; use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput}; use bevy_oxr::xr_input::interactions::{ @@ -19,7 +20,12 @@ fn main() { color_eyre::install().unwrap(); App::new() - .add_plugins(DefaultXrPlugins) + .add_plugins(DefaultXrPlugins { + app_info: XrAppInfo { + name: "Bevy OXR Globe Example".into(), + }, + ..default() + }) .add_plugins(LogDiagnosticsPlugin::default()) .add_systems(Startup, setup) .add_systems(Update, (proto_locomotion, pull_to_ground).chain()) diff --git a/examples/xr.rs b/examples/xr.rs index 79e1c90..b38d79a 100644 --- a/examples/xr.rs +++ b/examples/xr.rs @@ -2,6 +2,7 @@ use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}; use bevy::prelude::*; use bevy::transform::components::Transform; +use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::input::XrInput; use bevy_oxr::resources::{XrFrameState, XrSession}; @@ -25,7 +26,12 @@ fn main() { info!("Running `openxr-6dof` skill"); App::new() - .add_plugins(DefaultXrPlugins) + .add_plugins(DefaultXrPlugins { + app_info: XrAppInfo { + name: "Bevy OXR Example".into(), + }, + ..default() + }) //.add_plugins(OpenXrDebugRenderer) //new debug renderer adds gizmos to .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) diff --git a/src/graphics/extensions.rs b/src/graphics/extensions.rs new file mode 100644 index 0000000..908648a --- /dev/null +++ b/src/graphics/extensions.rs @@ -0,0 +1,242 @@ +use openxr::ExtensionSet; +use std::ops; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct XrExtensions(ExtensionSet); +impl XrExtensions { + pub fn raw_mut(&mut self) -> &mut ExtensionSet { + &mut self.0 + } + pub fn raw(&self) -> &ExtensionSet { + &self.0 + } + // pub fn enable_fb_passthrough(&mut self) -> &mut Self { + // self.0.fb_passthrough = true; + // self + // } + // pub fn disable_fb_passthrough(&mut self) -> &mut Self { + // self.0.fb_passthrough = false; + // self + // } + pub fn enable_hand_tracking(&mut self) -> &mut Self { + self.0.ext_hand_tracking = true; + self + } + pub fn disable_hand_tracking(&mut self) -> &mut Self { + self.0.ext_hand_tracking = false; + self + } +} +impl From for XrExtensions { + fn from(value: ExtensionSet) -> Self { + Self(value) + } +} +impl From for ExtensionSet { + fn from(val: XrExtensions) -> Self { + val.0 + } +} +impl Default for XrExtensions { + fn default() -> Self { + let mut exts = ExtensionSet::default(); + exts.ext_hand_tracking = true; + Self(exts) + } +} +impl ops::BitAnd for XrExtensions { + type Output = Self; + + fn bitand(self, rhs: Self) -> Self::Output { + let mut out = ExtensionSet::default(); + out.almalence_digital_lens_control = + self.0.almalence_digital_lens_control && rhs.0.almalence_digital_lens_control; + out.epic_view_configuration_fov = + self.0.epic_view_configuration_fov && rhs.0.epic_view_configuration_fov; + out.ext_performance_settings = + self.0.ext_performance_settings && rhs.0.ext_performance_settings; + out.ext_thermal_query = self.0.ext_thermal_query && rhs.0.ext_thermal_query; + out.ext_debug_utils = self.0.ext_debug_utils && rhs.0.ext_debug_utils; + out.ext_eye_gaze_interaction = + self.0.ext_eye_gaze_interaction && rhs.0.ext_eye_gaze_interaction; + out.ext_view_configuration_depth_range = + self.0.ext_view_configuration_depth_range && rhs.0.ext_view_configuration_depth_range; + out.ext_conformance_automation = + self.0.ext_conformance_automation && rhs.0.ext_conformance_automation; + out.ext_hand_tracking = self.0.ext_hand_tracking && rhs.0.ext_hand_tracking; + out.ext_dpad_binding = self.0.ext_dpad_binding && rhs.0.ext_dpad_binding; + out.ext_hand_joints_motion_range = + self.0.ext_hand_joints_motion_range && rhs.0.ext_hand_joints_motion_range; + out.ext_samsung_odyssey_controller = + self.0.ext_samsung_odyssey_controller && rhs.0.ext_samsung_odyssey_controller; + out.ext_hp_mixed_reality_controller = + self.0.ext_hp_mixed_reality_controller && rhs.0.ext_hp_mixed_reality_controller; + out.ext_palm_pose = self.0.ext_palm_pose && rhs.0.ext_palm_pose; + out.ext_uuid = self.0.ext_uuid && rhs.0.ext_uuid; + out.extx_overlay = self.0.extx_overlay && rhs.0.extx_overlay; + out.fb_composition_layer_image_layout = + self.0.fb_composition_layer_image_layout && rhs.0.fb_composition_layer_image_layout; + out.fb_composition_layer_alpha_blend = + self.0.fb_composition_layer_alpha_blend && rhs.0.fb_composition_layer_alpha_blend; + out.fb_swapchain_update_state = + self.0.fb_swapchain_update_state && rhs.0.fb_swapchain_update_state; + out.fb_composition_layer_secure_content = + self.0.fb_composition_layer_secure_content && rhs.0.fb_composition_layer_secure_content; + out.fb_display_refresh_rate = + self.0.fb_display_refresh_rate && rhs.0.fb_display_refresh_rate; + out.fb_color_space = self.0.fb_color_space && rhs.0.fb_color_space; + out.fb_hand_tracking_mesh = self.0.fb_hand_tracking_mesh && rhs.0.fb_hand_tracking_mesh; + out.fb_hand_tracking_aim = self.0.fb_hand_tracking_aim && rhs.0.fb_hand_tracking_aim; + out.fb_hand_tracking_capsules = + self.0.fb_hand_tracking_capsules && rhs.0.fb_hand_tracking_capsules; + out.fb_spatial_entity = self.0.fb_spatial_entity && rhs.0.fb_spatial_entity; + out.fb_foveation = self.0.fb_foveation && rhs.0.fb_foveation; + out.fb_foveation_configuration = + self.0.fb_foveation_configuration && rhs.0.fb_foveation_configuration; + out.fb_keyboard_tracking = self.0.fb_keyboard_tracking && rhs.0.fb_keyboard_tracking; + out.fb_triangle_mesh = self.0.fb_triangle_mesh && rhs.0.fb_triangle_mesh; + out.fb_passthrough = self.0.fb_passthrough && rhs.0.fb_passthrough; + out.fb_render_model = self.0.fb_render_model && rhs.0.fb_render_model; + out.fb_spatial_entity_query = + self.0.fb_spatial_entity_query && rhs.0.fb_spatial_entity_query; + out.fb_spatial_entity_storage = + self.0.fb_spatial_entity_storage && rhs.0.fb_spatial_entity_storage; + out.fb_foveation_vulkan = self.0.fb_foveation_vulkan && rhs.0.fb_foveation_vulkan; + out.fb_swapchain_update_state_opengl_es = + self.0.fb_swapchain_update_state_opengl_es && rhs.0.fb_swapchain_update_state_opengl_es; + out.fb_swapchain_update_state_vulkan = + self.0.fb_swapchain_update_state_vulkan && rhs.0.fb_swapchain_update_state_vulkan; + out.fb_space_warp = self.0.fb_space_warp && rhs.0.fb_space_warp; + out.fb_scene = self.0.fb_scene && rhs.0.fb_scene; + out.fb_spatial_entity_container = + self.0.fb_spatial_entity_container && rhs.0.fb_spatial_entity_container; + out.fb_passthrough_keyboard_hands = + self.0.fb_passthrough_keyboard_hands && rhs.0.fb_passthrough_keyboard_hands; + out.fb_composition_layer_settings = + self.0.fb_composition_layer_settings && rhs.0.fb_composition_layer_settings; + out.htc_vive_cosmos_controller_interaction = self.0.htc_vive_cosmos_controller_interaction + && rhs.0.htc_vive_cosmos_controller_interaction; + out.htc_facial_tracking = self.0.htc_facial_tracking && rhs.0.htc_facial_tracking; + out.htc_vive_focus3_controller_interaction = self.0.htc_vive_focus3_controller_interaction + && rhs.0.htc_vive_focus3_controller_interaction; + out.htc_hand_interaction = self.0.htc_hand_interaction && rhs.0.htc_hand_interaction; + out.htc_vive_wrist_tracker_interaction = + self.0.htc_vive_wrist_tracker_interaction && rhs.0.htc_vive_wrist_tracker_interaction; + out.htcx_vive_tracker_interaction = + self.0.htcx_vive_tracker_interaction && rhs.0.htcx_vive_tracker_interaction; + out.huawei_controller_interaction = + self.0.huawei_controller_interaction && rhs.0.huawei_controller_interaction; + out.khr_composition_layer_cube = + self.0.khr_composition_layer_cube && rhs.0.khr_composition_layer_cube; + out.khr_composition_layer_depth = + self.0.khr_composition_layer_depth && rhs.0.khr_composition_layer_depth; + out.khr_vulkan_swapchain_format_list = + self.0.khr_vulkan_swapchain_format_list && rhs.0.khr_vulkan_swapchain_format_list; + out.khr_composition_layer_cylinder = + self.0.khr_composition_layer_cylinder && rhs.0.khr_composition_layer_cylinder; + out.khr_composition_layer_equirect = + self.0.khr_composition_layer_equirect && rhs.0.khr_composition_layer_equirect; + out.khr_opengl_enable = self.0.khr_opengl_enable && rhs.0.khr_opengl_enable; + out.khr_opengl_es_enable = self.0.khr_opengl_es_enable && rhs.0.khr_opengl_es_enable; + out.khr_vulkan_enable = self.0.khr_vulkan_enable && rhs.0.khr_vulkan_enable; + out.khr_visibility_mask = self.0.khr_visibility_mask && rhs.0.khr_visibility_mask; + out.khr_composition_layer_color_scale_bias = self.0.khr_composition_layer_color_scale_bias + && rhs.0.khr_composition_layer_color_scale_bias; + out.khr_convert_timespec_time = + self.0.khr_convert_timespec_time && rhs.0.khr_convert_timespec_time; + out.khr_loader_init = self.0.khr_loader_init && rhs.0.khr_loader_init; + out.khr_vulkan_enable2 = self.0.khr_vulkan_enable2 && rhs.0.khr_vulkan_enable2; + out.khr_composition_layer_equirect2 = + self.0.khr_composition_layer_equirect2 && rhs.0.khr_composition_layer_equirect2; + out.khr_binding_modification = + self.0.khr_binding_modification && rhs.0.khr_binding_modification; + out.khr_swapchain_usage_input_attachment_bit = + self.0.khr_swapchain_usage_input_attachment_bit + && rhs.0.khr_swapchain_usage_input_attachment_bit; + out.meta_vulkan_swapchain_create_info = + self.0.meta_vulkan_swapchain_create_info && rhs.0.meta_vulkan_swapchain_create_info; + out.meta_performance_metrics = + self.0.meta_performance_metrics && rhs.0.meta_performance_metrics; + out.ml_ml2_controller_interaction = + self.0.ml_ml2_controller_interaction && rhs.0.ml_ml2_controller_interaction; + out.mnd_headless = self.0.mnd_headless && rhs.0.mnd_headless; + out.mnd_swapchain_usage_input_attachment_bit = + self.0.mnd_swapchain_usage_input_attachment_bit + && rhs.0.mnd_swapchain_usage_input_attachment_bit; + out.mndx_egl_enable = self.0.mndx_egl_enable && rhs.0.mndx_egl_enable; + out.msft_unbounded_reference_space = + self.0.msft_unbounded_reference_space && rhs.0.msft_unbounded_reference_space; + out.msft_spatial_anchor = self.0.msft_spatial_anchor && rhs.0.msft_spatial_anchor; + out.msft_spatial_graph_bridge = + self.0.msft_spatial_graph_bridge && rhs.0.msft_spatial_graph_bridge; + out.msft_hand_interaction = self.0.msft_hand_interaction && rhs.0.msft_hand_interaction; + out.msft_hand_tracking_mesh = + self.0.msft_hand_tracking_mesh && rhs.0.msft_hand_tracking_mesh; + out.msft_secondary_view_configuration = + self.0.msft_secondary_view_configuration && rhs.0.msft_secondary_view_configuration; + out.msft_first_person_observer = + self.0.msft_first_person_observer && rhs.0.msft_first_person_observer; + out.msft_controller_model = self.0.msft_controller_model && rhs.0.msft_controller_model; + out.msft_composition_layer_reprojection = + self.0.msft_composition_layer_reprojection && rhs.0.msft_composition_layer_reprojection; + out.msft_spatial_anchor_persistence = + self.0.msft_spatial_anchor_persistence && rhs.0.msft_spatial_anchor_persistence; + out.oculus_audio_device_guid = + self.0.oculus_audio_device_guid && rhs.0.oculus_audio_device_guid; + out.ultraleap_hand_tracking_forearm = + self.0.ultraleap_hand_tracking_forearm && rhs.0.ultraleap_hand_tracking_forearm; + out.valve_analog_threshold = self.0.valve_analog_threshold && rhs.0.valve_analog_threshold; + out.varjo_quad_views = self.0.varjo_quad_views && rhs.0.varjo_quad_views; + out.varjo_foveated_rendering = + self.0.varjo_foveated_rendering && rhs.0.varjo_foveated_rendering; + out.varjo_composition_layer_depth_test = + self.0.varjo_composition_layer_depth_test && rhs.0.varjo_composition_layer_depth_test; + out.varjo_environment_depth_estimation = + self.0.varjo_environment_depth_estimation && rhs.0.varjo_environment_depth_estimation; + out.varjo_marker_tracking = self.0.varjo_marker_tracking && rhs.0.varjo_marker_tracking; + out.varjo_view_offset = self.0.varjo_view_offset && rhs.0.varjo_view_offset; + and_android_only_exts(&self, &rhs, &mut out); + and_windows_only_exts(&self, &rhs, &mut out); + for ext in self.0.other { + if rhs.0.other.contains(&ext) { + out.other.push(ext); + } + } + Self(out) + } +} + +#[cfg(not(target_os = "android"))] +fn and_android_only_exts(lhs: &XrExtensions, rhs: &XrExtensions, out: &mut ExtensionSet) {} +#[cfg(not(windows))] +fn and_windows_only_exts(lhs: &XrExtensions, rhs: &XrExtensions, out: &mut ExtensionSet) {} +#[cfg(target_os = "android")] +fn and_android_only_exts(lhs: &XrExtensions, rhs: &XrExtensions, out: &mut ExtensionSet) { + out.oculus_android_session_state_enable = + lhs.0.oculus_android_session_state_enable && rhs.0.oculus_android_session_state_enable; + out.khr_loader_init_android = lhs.0.khr_loader_init_android && rhs.0.khr_loader_init_android; + out.fb_android_surface_swapchain_create = + lhs.0.fb_android_surface_swapchain_create && rhs.0.fb_android_surface_swapchain_create; + out.fb_swapchain_update_state_android_surface = lhs.0.fb_swapchain_update_state_android_surface + && rhs.0.fb_swapchain_update_state_android_surface; + out.khr_android_thread_settings = + lhs.0.khr_android_thread_settings && rhs.0.khr_android_thread_settings; + out.khr_android_surface_swapchain = + lhs.0.khr_android_surface_swapchain && rhs.0.khr_android_surface_swapchain; + out.khr_android_create_instance = + lhs.0.khr_android_create_instance && rhs.0.khr_android_create_instance; +} +#[cfg(windows)] +fn and_windows_only_exts(lhs: &XrExtensions, rhs: &XrExtensions, out: &mut ExtensionSet) { + out.ext_win32_appcontainer_compatible = + lhs.0.ext_win32_appcontainer_compatible && rhs.0.ext_win32_appcontainer_compatible; + out.khr_d3d11_enable = lhs.0.khr_d3d11_enable && rhs.0.khr_d3d11_enable; + out.khr_d3d12_enable = lhs.0.khr_d3d12_enable && rhs.0.khr_d3d12_enable; + out.khr_win32_convert_performance_counter_time = + lhs.0.khr_win32_convert_performance_counter_time + && rhs.0.khr_win32_convert_performance_counter_time; + out.msft_perception_anchor_interop = + lhs.0.msft_perception_anchor_interop && rhs.0.msft_perception_anchor_interop; + out.msft_holographic_window_attachment = + lhs.0.msft_holographic_window_attachment && rhs.0.msft_holographic_window_attachment; +} diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index f881148..0aa3dfc 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -1,3 +1,4 @@ +pub mod extensions; mod vulkan; use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}; @@ -12,8 +13,37 @@ use crate::resources::{ use openxr as xr; +use self::extensions::XrExtensions; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum XrPreferdBlendMode { + Opaque, + Additive, + AlphaBlend, +} +impl Default for XrPreferdBlendMode { + fn default() -> Self { + Self::Opaque + } +} + +#[derive(Clone, Debug)] +pub struct XrAppInfo { + pub name: String, +} +impl Default for XrAppInfo { + fn default() -> Self { + Self { + name: "Ambient".into(), + } + } +} + pub fn initialize_xr_graphics( window: Option, + reqeusted_extensions: XrExtensions, + prefered_blend_mode: XrPreferdBlendMode, + app_info: XrAppInfo, ) -> anyhow::Result<( RenderDevice, RenderQueue, @@ -32,13 +62,13 @@ pub fn initialize_xr_graphics( XrViews, XrFrameState, )> { - vulkan::initialize_xr_graphics(window) + vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info) } pub fn xr_entry() -> anyhow::Result { - #[cfg(feature = "linked")] + #[cfg(windows)] let entry = Ok(xr::Entry::linked()); - #[cfg(not(feature = "linked"))] + #[cfg(not(windows))] let entry = unsafe { xr::Entry::load().map_err(|e| anyhow::anyhow!(e)) }; entry } diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index 55c5609..ba9814e 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -10,7 +10,9 @@ use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, Ren use bevy::window::RawHandleWrapper; use openxr as xr; use wgpu::Instance; +use xr::EnvironmentBlendMode; +use crate::graphics::extensions::XrExtensions; use crate::input::XrInput; use crate::resources::{ Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, @@ -18,10 +20,13 @@ use crate::resources::{ }; use crate::VIEW_TYPE; +use super::{XrAppInfo, XrPreferdBlendMode}; + pub fn initialize_xr_graphics( window: Option, - // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load - // any extension at some point + reqeusted_extensions: XrExtensions, + prefered_blend_mode: XrPreferdBlendMode, + app_info: XrAppInfo, ) -> anyhow::Result<( RenderDevice, RenderQueue, @@ -39,8 +44,6 @@ pub fn initialize_xr_graphics( XrInput, XrViews, XrFrameState, - // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load - // any extension at some point )> { use wgpu_hal::{api::Vulkan as V, Api}; @@ -49,26 +52,25 @@ pub fn initialize_xr_graphics( #[cfg(target_os = "android")] xr_entry.initialize_android_loader()?; - let available_extensions = xr_entry.enumerate_extensions()?; - assert!(available_extensions.khr_vulkan_enable2); + let available_extensions: XrExtensions = xr_entry.enumerate_extensions()?.into(); + assert!(available_extensions.raw().khr_vulkan_enable2); info!("available xr exts: {:#?}", available_extensions); - let mut enabled_extensions = xr::ExtensionSet::default(); + let mut enabled_extensions: xr::ExtensionSet = + (available_extensions & reqeusted_extensions).into(); enabled_extensions.khr_vulkan_enable2 = true; #[cfg(target_os = "android")] { enabled_extensions.khr_android_create_instance = true; } - enabled_extensions.ext_hand_tracking = available_extensions.ext_hand_tracking; - // enabled_extensions.ext_hand_joints_motion_range = available_extensions.ext_hand_joints_motion_range; - let available_layers = xr_entry.enumerate_layers()?; info!("available xr layers: {:#?}", available_layers); let xr_instance = xr_entry.create_instance( &xr::ApplicationInfo { - application_name: "Ambient", + application_name: &app_info.name, + engine_name: "Bevy", ..Default::default() }, &enabled_extensions, @@ -90,7 +92,22 @@ pub fn initialize_xr_graphics( } ); - let blend_mode = xr_instance.enumerate_environment_blend_modes(xr_system_id, VIEW_TYPE)?[0]; + let blend_modes = xr_instance.enumerate_environment_blend_modes(xr_system_id, VIEW_TYPE)?; + let blend_mode: EnvironmentBlendMode = match prefered_blend_mode { + XrPreferdBlendMode::Opaque if blend_modes.contains(&EnvironmentBlendMode::OPAQUE) => { + EnvironmentBlendMode::OPAQUE + } + XrPreferdBlendMode::Additive if blend_modes.contains(&EnvironmentBlendMode::ADDITIVE) => { + EnvironmentBlendMode::ADDITIVE + } + XrPreferdBlendMode::AlphaBlend + if blend_modes.contains(&EnvironmentBlendMode::ALPHA_BLEND) => + { + EnvironmentBlendMode::ALPHA_BLEND + } + _ => EnvironmentBlendMode::OPAQUE, + }; + #[cfg(not(target_os = "android"))] let vk_target_version = vk::make_api_version(0, 1, 2, 0); @@ -131,7 +148,7 @@ pub fn initialize_xr_graphics( let vk_instance = unsafe { let extensions_cchar: Vec<_> = extensions.iter().map(|s| s.as_ptr()).collect(); - let app_name = CString::new("Ambient")?; + let app_name = CString::new(app_info.name)?; let vk_app_info = vk::ApplicationInfo::builder() .application_name(&app_name) .application_version(1) @@ -409,8 +426,6 @@ pub fn initialize_xr_graphics( should_render: true, }) .into(), - // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load - // any extension at some point )) } diff --git a/src/input.rs b/src/input.rs index 78cad05..33b1785 100644 --- a/src/input.rs +++ b/src/input.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use bevy::prelude::*; use openxr as xr; +use xr::{FrameState, FrameWaiter, ViewConfigurationType}; #[derive(Clone, Resource)] pub struct XrInput { @@ -14,13 +15,12 @@ pub struct XrInput { } impl XrInput { - pub fn new(_instance: xr::Instance, session: xr::Session) -> xr::Result { - // let action_set = instance.create_action_set("input", "input pose information", 0)?; - // let left_hand_subaction_path = instance.string_to_path("/user/hand/left").unwrap(); + pub fn new( + instance: xr::Instance, + session: xr::Session, + // frame_state: &FrameState, + ) -> xr::Result { // let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap(); - // let left_hand_grip_pose_path = instance - // .string_to_path("/user/hand/left/input/grip/pose") - // .unwrap(); // let right_hand_grip_pose_path = instance // .string_to_path("/user/hand/right/input/grip/pose") // .unwrap(); @@ -49,11 +49,23 @@ impl XrInput { // left_hand_subaction_path, // xr::Posef::IDENTITY, // )?; + let stage = session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?; - let head = session - .create_reference_space(xr::ReferenceSpaceType::VIEW, xr::Posef::IDENTITY) - .unwrap(); + let head = + session.create_reference_space(xr::ReferenceSpaceType::VIEW, xr::Posef::IDENTITY)?; + // let y = stage + // .locate(&head, frame_state.predicted_display_time).unwrap() + // .pose + // .position + // .y; + // let local = session.create_reference_space( + // xr::ReferenceSpaceType::LOCAL, + // xr::Posef { + // position: xr::Vector3f { x: 0.0, y, z: 0.0 }, + // orientation: xr::Quaternionf::IDENTITY, + // }, + // ).unwrap(); //session.attach_action_sets(&[&action_set])?; //session.attach_action_sets(&[])?; Ok(Self { diff --git a/src/lib.rs b/src/lib.rs index 3957e5e..9a77e87 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ -mod graphics; +pub mod graphics; pub mod input; +// pub mod passthrough; pub mod resource_macros; pub mod resources; pub mod xr_init; @@ -10,33 +11,23 @@ use std::sync::{Arc, Mutex}; use crate::xr_init::RenderRestartPlugin; use crate::xr_input::hands::hand_tracking::DisableHandTracking; use crate::xr_input::oculus_touch::ActionSets; -use bevy::app::PluginGroupBuilder; -use bevy::ecs::system::{RunSystemOnce, SystemState}; +use bevy::app::{AppExit, PluginGroupBuilder}; +use bevy::ecs::system::SystemState; use bevy::prelude::*; -use bevy::render::camera::{ - CameraPlugin, ManualTextureView, ManualTextureViewHandle, ManualTextureViews, -}; -use bevy::render::globals::GlobalsPlugin; -use bevy::render::mesh::morph::MorphPlugin; -use bevy::render::mesh::MeshPlugin; +use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; -use bevy::render::render_asset::RenderAssetDependency; -use bevy::render::render_resource::ShaderLoader; -use bevy::render::renderer::{ - render_system, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, -}; +use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; -use bevy::render::view::{self, ViewPlugin, WindowRenderPlugin}; -use bevy::render::{color, primitives, Render, RenderApp, RenderPlugin, RenderSet}; +use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; +use graphics::extensions::XrExtensions; +use graphics::{XrAppInfo, XrPreferdBlendMode}; use input::XrInput; use openxr as xr; +// use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; use resources::*; use xr::FormFactor; -use xr_init::{ - init_non_xr_graphics, update_xr_stuff, xr_only, RenderCreationData, XrEnableRequest, - XrEnableStatus, XrRenderData, XrRenderUpdate, -}; +use xr_init::{xr_only, XrEnableStatus, XrRenderData}; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin}; @@ -48,12 +39,11 @@ pub const LEFT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHan pub const RIGHT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(3383858418); /// Adds OpenXR support to an App -pub struct OpenXrPlugin; - -impl Default for OpenXrPlugin { - fn default() -> Self { - OpenXrPlugin - } +#[derive(Default)] +pub struct OpenXrPlugin { + reqeusted_extensions: XrExtensions, + prefered_blend_mode: XrPreferdBlendMode, + app_info: XrAppInfo, } #[derive(Resource)] @@ -76,6 +66,9 @@ pub struct FutureXrResources( >, >, ); +// fn mr_test(mut commands: Commands, passthrough_layer: Option>) { +// commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); +// } impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { @@ -84,7 +77,12 @@ impl Plugin for OpenXrPlugin { let primary_window = system_state.get(&app.world).get_single().ok().cloned(); #[cfg(not(target_arch = "wasm32"))] - match graphics::initialize_xr_graphics(primary_window.clone()) { + match graphics::initialize_xr_graphics( + primary_window.clone(), + self.reqeusted_extensions.clone(), + self.prefered_blend_mode, + self.app_info.clone(), + ) { Ok(( device, queue, @@ -149,6 +147,7 @@ impl Plugin for OpenXrPlugin { app.insert_resource(XrEnableStatus::Disabled); } } + // app.add_systems(PreUpdate, mr_test); #[cfg(target_arch = "wasm32")] { app.add_plugins(RenderPlugin::default()); @@ -180,6 +179,23 @@ impl Plugin for OpenXrPlugin { } else { app.insert_resource(DisableHandTracking::Both); } + // let passthrough = data.xr_instance.exts().fb_passthrough.is_some() + // && supports_passthrough( + // &data.xr_instance, + // data.xr_instance + // .system(FormFactor::HEAD_MOUNTED_DISPLAY) + // .unwrap(), + // ) + // .is_ok_and(|v| v); + // if passthrough { + // info!("Passthrough!"); + // let (pl, p) = start_passthrough(&data); + // app.insert_resource(pl); + // app.insert_resource(p); + // // if !app.world.contains_resource::() { + // // info!("ClearColor!"); + // // } + // } let (left, right) = data.xr_swapchain.get_render_views(); let left = ManualTextureView { @@ -225,7 +241,12 @@ impl Plugin for OpenXrPlugin { } } -pub struct DefaultXrPlugins; +#[derive(Default)] +pub struct DefaultXrPlugins { + pub reqeusted_extensions: XrExtensions, + pub prefered_blend_mode: XrPreferdBlendMode, + pub app_info: XrAppInfo, +} impl PluginGroup for DefaultXrPlugins { fn build(self) -> PluginGroupBuilder { @@ -233,7 +254,11 @@ impl PluginGroup for DefaultXrPlugins { .build() .disable::() .disable::() - .add_before::(OpenXrPlugin) + .add_before::(OpenXrPlugin { + prefered_blend_mode: self.prefered_blend_mode, + reqeusted_extensions: self.reqeusted_extensions, + app_info: self.app_info.clone(), + }) .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) .add_before::(RenderRestartPlugin) .add(HandEmulationPlugin) @@ -241,7 +266,9 @@ impl PluginGroup for DefaultXrPlugins { .set(WindowPlugin { #[cfg(not(target_os = "android"))] primary_window: Some(Window { + transparent: true, present_mode: PresentMode::AutoNoVsync, + title: self.app_info.name.clone(), ..default() }), #[cfg(target_os = "android")] @@ -264,6 +291,7 @@ pub fn xr_begin_frame( swapchain: Res, views: Res, input: Res, + mut app_exit: EventWriter, ) { { let _span = info_span!("xr_poll_events"); @@ -282,8 +310,12 @@ pub fn xr_begin_frame( xr::SessionState::STOPPING => { session.end().unwrap(); session_running.store(false, std::sync::atomic::Ordering::Relaxed); + app_exit.send(AppExit); + } + xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { + app_exit.send(AppExit); + return; } - xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => return, _ => {} } } @@ -361,6 +393,7 @@ pub fn end_frame( swapchain: Res, resolution: Res, environment_blend_mode: Res, + // passthrough_layer: Option>, ) { { let _span = info_span!("xr_release_image").entered(); @@ -370,10 +403,11 @@ pub fn end_frame( let _span = info_span!("xr_end_frame").entered(); let result = swapchain.end( xr_frame_state.lock().unwrap().predicted_display_time, - &*views.lock().unwrap(), + &views.lock().unwrap(), &input.stage, **resolution, **environment_blend_mode, + // passthrough_layer.map(|p| p.into_inner()), ); match result { Ok(_) => {} diff --git a/src/resources.rs b/src/resources.rs index 2715027..a798955 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -1,6 +1,7 @@ use std::sync::atomic::AtomicBool; use std::sync::Mutex; +// use crate::passthrough::XrPassthroughLayer; use crate::resource_macros::*; use bevy::prelude::*; use openxr as xr; @@ -58,6 +59,7 @@ impl Swapchain { stage: &xr::Space, resolution: UVec2, environment_blend_mode: xr::EnvironmentBlendMode, + // passthrough_layer: Option<&XrPassthroughLayer>, ) -> xr::Result<()> { match self { Swapchain::Vulkan(swapchain) => swapchain.end( @@ -66,6 +68,7 @@ impl Swapchain { stage, resolution, environment_blend_mode, + // passthrough_layer, ), } } @@ -125,6 +128,7 @@ impl SwapchainInner { stage: &xr::Space, resolution: UVec2, environment_blend_mode: xr::EnvironmentBlendMode, + // passthrough_layer: Option<&XrPassthroughLayer>, ) -> xr::Result<()> { let rect = xr::Rect2Di { offset: xr::Offset2Di { x: 0, y: 0 }, @@ -138,6 +142,51 @@ impl SwapchainInner { warn!("views are len of 0"); return Ok(()); } + // match passthrough_layer { + // Some(pass) => { + // // info!("Rendering with pass through"); + // let passthrough_layer = xr::sys::CompositionLayerPassthroughFB { + // ty: CompositionLayerPassthroughFB::TYPE, + // next: ptr::null(), + // flags: CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA, + // space: xr::sys::Space::NULL, + // layer_handle: pass.0, + // }; + // self.stream.lock().unwrap().end( + // predicted_display_time, + // environment_blend_mode, + // &[ + // &xr::CompositionLayerProjection::new() + // .layer_flags(CompositionLayerFlags::UNPREMULTIPLIED_ALPHA) + // .space(stage) + // .views(&[ + // xr::CompositionLayerProjectionView::new() + // .pose(views[0].pose) + // .fov(views[0].fov) + // .sub_image( + // xr::SwapchainSubImage::new() + // .swapchain(&swapchain) + // .image_array_index(0) + // .image_rect(rect), + // ), + // xr::CompositionLayerProjectionView::new() + // .pose(views[1].pose) + // .fov(views[1].fov) + // .sub_image( + // xr::SwapchainSubImage::new() + // .swapchain(&swapchain) + // .image_array_index(1) + // .image_rect(rect), + // ), + // ]), + // unsafe { + // &*(&passthrough_layer as *const _ as *const CompositionLayerBase) + // }, + // ], + // ) + // } + + // None => self.stream.lock().unwrap().end( predicted_display_time, environment_blend_mode, @@ -162,5 +211,6 @@ impl SwapchainInner { ), ])], ) + // } } } diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 47802fd..9964ce6 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -79,7 +79,7 @@ pub fn setup_oxr_actions(world: &mut World) { actions.insert(action_name, typed_action); for (device_path, bindings) in action.bindings.into_iter() { for b in bindings { - info!("binding {} to {}", action_name, b); + // info!("binding {} to {}", action_name, b); action_bindings .entry((set_name, action_name)) .or_default() @@ -117,7 +117,6 @@ pub fn setup_oxr_actions(world: &mut World) { .map(move |(dev, bindings)| (action, dev, bindings)) }) .map(|(action, dev, bindings)| { - info!("Hi"); ( dev, bindings @@ -136,7 +135,6 @@ pub fn setup_oxr_actions(world: &mut World) { b_indings.entry(dev).or_default().append(&mut bindings); } for (dev, bindings) in b_indings.into_iter() { - info!(dev); instance .suggest_interaction_profile_bindings(instance.string_to_path(dev).unwrap(), &bindings) .expect("Unable to suggest interaction bindings!"); @@ -305,7 +303,7 @@ impl XrActionSets { &self, action_set: &'static str, action_name: &'static str, - ) -> Result<&Action,ActionError> { + ) -> Result<&Action, ActionError> { let action = self .sets .get(action_set) @@ -322,7 +320,7 @@ impl XrActionSets { &self, action_set: &'static str, action_name: &'static str, - ) -> Result<&Action,ActionError> { + ) -> Result<&Action, ActionError> { let action = self .sets .get(action_set) @@ -339,7 +337,7 @@ impl XrActionSets { &self, action_set: &'static str, action_name: &'static str, - ) -> Result<&Action,ActionError> { + ) -> Result<&Action, ActionError> { let action = self .sets .get(action_set) @@ -356,7 +354,7 @@ impl XrActionSets { &self, action_set: &'static str, action_name: &'static str, - ) -> Result<&Action,ActionError> { + ) -> Result<&Action, ActionError> { let action = self .sets .get(action_set) diff --git a/src/xr_input/hands/common.rs b/src/xr_input/hands/common.rs index 2c88b08..104c6b3 100644 --- a/src/xr_input/hands/common.rs +++ b/src/xr_input/hands/common.rs @@ -3,9 +3,9 @@ use bevy::prelude::{ Query, Resource, SpatialBundle, Startup, Transform, }; -use crate::xr_input::{Hand, trackers::OpenXRTracker}; +use crate::xr_input::{trackers::OpenXRTracker, Hand}; -use super::{HandBone, BoneTrackingStatus}; +use super::{BoneTrackingStatus, HandBone}; /// add debug renderer for controllers #[derive(Default)] diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index 05d6d95..1af55df 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -1,16 +1,13 @@ use bevy::prelude::*; use openxr::{HandTracker, Result, SpaceLocationFlags}; +use super::common::HandBoneRadius; use crate::{ input::XrInput, - resources::{XrFrameState, XrSession}, - xr_input::{ - hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv, - Vec3Conv, - }, xr_init::xr_only, + xr_init::xr_only, + xr_input::{hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv, Vec3Conv}, }; -use super::common::HandBoneRadius; use super::BoneTrackingStatus; @@ -126,9 +123,11 @@ impl Plugin for HandTrackingPlugin { app.add_systems( PreUpdate, ( - update_hand_bones.run_if(|dh: Option>| { - !dh.is_some_and(|v| *v == DisableHandTracking::Both) - }).run_if(xr_only()), + update_hand_bones + .run_if(|dh: Option>| { + !dh.is_some_and(|v| *v == DisableHandTracking::Both) + }) + .run_if(xr_only()), update_tracking_state_on_disable, ), ); diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs index db760dc..06eb5c1 100644 --- a/src/xr_input/hands/mod.rs +++ b/src/xr_input/hands/mod.rs @@ -2,9 +2,9 @@ use bevy::{app::PluginGroupBuilder, prelude::*}; use self::{emulated::HandEmulationPlugin, hand_tracking::HandTrackingPlugin}; +pub mod common; pub mod emulated; pub mod hand_tracking; -pub mod common; pub struct XrHandPlugins; diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index 50268c8..540f2e2 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -11,7 +11,7 @@ pub mod xr_camera; use crate::resources::{XrInstance, XrSession}; use crate::xr_begin_frame; -use crate::xr_init::{xr_only, XrPostSetup, XrSetup}; +use crate::xr_init::{xr_only, XrPostSetup, XrSetup, XrPreSetup}; use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; @@ -30,7 +30,7 @@ use bevy::utils::HashMap; use openxr::Binding; use self::actions::{setup_oxr_actions, OpenXrActionsPlugin}; -use self::oculus_touch::{post_action_setup_oculus_controller, ActionSets}; +use self::oculus_touch::{post_action_setup_oculus_controller, ActionSets, init_subaction_path}; use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, @@ -77,6 +77,7 @@ impl Plugin for OpenXrInput { .after(TransformSystem::TransformPropagate) .before(VisibilitySystems::UpdatePerspectiveFrusta), ); + app.add_systems(XrPreSetup, init_subaction_path); app.add_systems(XrSetup, setup_xr_cameras); } } diff --git a/src/xr_input/oculus_touch.rs b/src/xr_input/oculus_touch.rs index 6d416bb..6f405f8 100644 --- a/src/xr_input/oculus_touch.rs +++ b/src/xr_input/oculus_touch.rs @@ -2,7 +2,7 @@ use crate::input::XrInput; use crate::resources::{XrInstance, XrSession}; use crate::xr_input::controllers::Handed; use crate::xr_input::Hand; -use bevy::prelude::{Commands, Res, ResMut, Resource}; +use bevy::prelude::{default, Commands, Res, ResMut, Resource}; use openxr::{ ActionSet, AnyGraphics, FrameState, Instance, Path, Posef, Session, Space, SpaceLocation, SpaceVelocity, @@ -50,7 +50,6 @@ pub fn setup_oculus_controller( action_sets: ResMut, ) { let oculus_controller = OculusController::new(action_sets).unwrap(); - init_subaction_path(&instance); commands.insert_resource(oculus_controller); } @@ -65,10 +64,10 @@ pub struct OculusControllerRef<'a> { xr_input: &'a XrInput, } -static RIGHT_SUBACTION_PATH: OnceLock = OnceLock::new(); -static LEFT_SUBACTION_PATH: OnceLock = OnceLock::new(); +pub static RIGHT_SUBACTION_PATH: OnceLock = OnceLock::new(); +pub static LEFT_SUBACTION_PATH: OnceLock = OnceLock::new(); -pub fn init_subaction_path(instance: &Instance) { +pub fn init_subaction_path(instance: Res) { let _ = LEFT_SUBACTION_PATH.set(instance.string_to_path("/user/hand/left").unwrap()); let _ = RIGHT_SUBACTION_PATH.set(instance.string_to_path("/user/hand/right").unwrap()); } @@ -82,7 +81,7 @@ pub fn subaction_path(hand: Hand) -> Path { impl OculusControllerRef<'_> { pub fn grip_space(&self, hand: Hand) -> (SpaceLocation, SpaceVelocity) { - match hand { + let d = match hand { Hand::Left => self .oculus_controller .grip_space @@ -103,11 +102,14 @@ impl OculusControllerRef<'_> { &self.xr_input.stage, self.frame_state.predicted_display_time, ), + }; + match d { + Ok(d) => d, + Err(_) => (SpaceLocation::default(), SpaceVelocity::default()), } - .unwrap() } pub fn aim_space(&self, hand: Hand) -> (SpaceLocation, SpaceVelocity) { - match hand { + let d = match hand { Hand::Left => self .oculus_controller .aim_space @@ -128,147 +130,213 @@ impl OculusControllerRef<'_> { &self.xr_input.stage, self.frame_state.predicted_display_time, ), + }; + match d { + Ok(d) => d, + Err(_) => (SpaceLocation::default(), SpaceVelocity::default()), } - .unwrap() } pub fn squeeze(&self, hand: Hand) -> f32 { - let action = &self + match &self .action_sets .get_action_f32("oculus_input", "squeeze") - .unwrap(); - action - .state(&self.session, subaction_path(hand)) .unwrap() - .current_state + .state(&self.session, subaction_path(hand)) + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn trigger(&self, hand: Hand) -> f32 { - self.action_sets + match self + .action_sets .get_action_f32("oculus_input", "trigger") .unwrap() .state(&self.session, subaction_path(hand)) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn trigger_touched(&self, hand: Hand) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "trigger_touched") .unwrap() .state(&self.session, subaction_path(hand)) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn x_button(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "x_button") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn x_button_touched(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "x_button_touch") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn y_button(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "y_button") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn y_button_touched(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "y_button_touch") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn menu_button(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "menu_button") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn a_button(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "a_button") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn a_button_touched(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "a_button_touch") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn b_button(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "b_button") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn b_button_touched(&self) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "b_button_touch") .unwrap() .state(&self.session, Path::NULL) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn thumbstick_touch(&self, hand: Hand) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "thumbstick_touch") .unwrap() .state(&self.session, subaction_path(hand)) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } pub fn thumbstick(&self, hand: Hand) -> Thumbstick { Thumbstick { - x: self + x: match self .action_sets .get_action_f32("oculus_input", "thumbstick_x") .unwrap() .state(&self.session, subaction_path(hand)) - .unwrap() - .current_state, - y: self + .map(|v| v.current_state) + { + Ok(v) => v, + Err(_) => default(), + }, + y: match self .action_sets .get_action_f32("oculus_input", "thumbstick_y") .unwrap() .state(&self.session, subaction_path(hand)) - .unwrap() - .current_state, - click: self + .map(|v| v.current_state) + { + Ok(v) => v, + Err(_) => default(), + }, + click: match self .action_sets .get_action_bool("oculus_input", "thumbstick_click") .unwrap() .state(&self.session, subaction_path(hand)) - .unwrap() - .current_state, + .map(|v| v.current_state) + { + Ok(v) => v, + Err(_) => default(), + }, } } pub fn thumbrest_touch(&self, hand: Hand) -> bool { - self.action_sets + match self + .action_sets .get_action_bool("oculus_input", "thumbrest_touch") .unwrap() .state(&self.session, subaction_path(hand)) - .unwrap() - .current_state + { + Ok(v) => v, + Err(_) => return default(), + } + .current_state } } From b2efa802cb2370cc6e67fa171f260df3f25baa68 Mon Sep 17 00:00:00 2001 From: Schmarni <51007916+Schmarni-Dev@users.noreply.github.com> Date: Mon, 1 Jan 2024 02:30:12 +0100 Subject: [PATCH 09/80] use openxr git version (#62) * fix problems with get version * a litle Recentering as a treat? * commit new Cargo.toml * re export path null so you don't have to import openxr yourself --- Cargo.toml | 15 ++++++++++++--- examples/android/Cargo.toml | 2 +- src/graphics/extensions.rs | 18 ++++++++++++++---- src/input.rs | 10 ++++++++-- src/xr_input/actions.rs | 2 ++ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c35974e..2836170 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,11 +25,20 @@ wgpu-core = { version = "0.17.1", features = ["vulkan"] } wgpu-hal = "0.17.1" [target.'cfg(windows)'.dependencies] -openxr = { version = "0.17.1", features = ["linked","static","mint"] } +openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = [ + "linked", + "static", + "mint", +] } [target.'cfg(all(target_family = "unix", not(target_arch = "wasm32")) )'.dependencies] -openxr = { version = "0.17.1", features = ["mint"] } +openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = [ + "mint", +] } [target.'cfg(all(not(target_family = "unix"), not(target_arch = "wasm32")))'.dependencies] -openxr = { version = "0.17.1", features = ["mint", "static"] } +openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = [ + "mint", + "static", +] } [dev-dependencies] bevy = "0.12" diff --git a/examples/android/Cargo.toml b/examples/android/Cargo.toml index 9f3f65a..bea6cb3 100644 --- a/examples/android/Cargo.toml +++ b/examples/android/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["rlib", "cdylib"] [dependencies] bevy_oxr.path = "../.." bevy = "0.12" -openxr = { git = "https://github.com/Ralith/openxrs", features = ["mint"] } +openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = ["mint"] } [profile.release] lto = "fat" diff --git a/src/graphics/extensions.rs b/src/graphics/extensions.rs index 908648a..2276b79 100644 --- a/src/graphics/extensions.rs +++ b/src/graphics/extensions.rs @@ -26,6 +26,14 @@ impl XrExtensions { self.0.ext_hand_tracking = false; self } + pub fn enable_local_floor(&mut self) -> &mut Self { + self.0.ext_local_floor = true; + self + } + pub fn disable_local_floor(&mut self) -> &mut Self { + self.0.ext_local_floor = false; + self + } } impl From for XrExtensions { fn from(value: ExtensionSet) -> Self { @@ -41,6 +49,7 @@ impl Default for XrExtensions { fn default() -> Self { let mut exts = ExtensionSet::default(); exts.ext_hand_tracking = true; + exts.ext_local_floor = true; Self(exts) } } @@ -49,6 +58,7 @@ impl ops::BitAnd for XrExtensions { fn bitand(self, rhs: Self) -> Self::Output { let mut out = ExtensionSet::default(); + out.ext_local_floor = self.0.ext_local_floor && rhs.0.ext_local_floor; out.almalence_digital_lens_control = self.0.almalence_digital_lens_control && rhs.0.almalence_digital_lens_control; out.epic_view_configuration_fov = @@ -73,7 +83,7 @@ impl ops::BitAnd for XrExtensions { self.0.ext_hp_mixed_reality_controller && rhs.0.ext_hp_mixed_reality_controller; out.ext_palm_pose = self.0.ext_palm_pose && rhs.0.ext_palm_pose; out.ext_uuid = self.0.ext_uuid && rhs.0.ext_uuid; - out.extx_overlay = self.0.extx_overlay && rhs.0.extx_overlay; + // out.extx_overlay = self.0.extx_overlay && rhs.0.extx_overlay; out.fb_composition_layer_image_layout = self.0.fb_composition_layer_image_layout && rhs.0.fb_composition_layer_image_layout; out.fb_composition_layer_alpha_blend = @@ -122,8 +132,8 @@ impl ops::BitAnd for XrExtensions { out.htc_hand_interaction = self.0.htc_hand_interaction && rhs.0.htc_hand_interaction; out.htc_vive_wrist_tracker_interaction = self.0.htc_vive_wrist_tracker_interaction && rhs.0.htc_vive_wrist_tracker_interaction; - out.htcx_vive_tracker_interaction = - self.0.htcx_vive_tracker_interaction && rhs.0.htcx_vive_tracker_interaction; + // out.htcx_vive_tracker_interaction = + // self.0.htcx_vive_tracker_interaction && rhs.0.htcx_vive_tracker_interaction; out.huawei_controller_interaction = self.0.huawei_controller_interaction && rhs.0.huawei_controller_interaction; out.khr_composition_layer_cube = @@ -163,7 +173,7 @@ impl ops::BitAnd for XrExtensions { out.mnd_swapchain_usage_input_attachment_bit = self.0.mnd_swapchain_usage_input_attachment_bit && rhs.0.mnd_swapchain_usage_input_attachment_bit; - out.mndx_egl_enable = self.0.mndx_egl_enable && rhs.0.mndx_egl_enable; + // out.mndx_egl_enable = self.0.mndx_egl_enable && rhs.0.mndx_egl_enable; out.msft_unbounded_reference_space = self.0.msft_unbounded_reference_space && rhs.0.msft_unbounded_reference_space; out.msft_spatial_anchor = self.0.msft_spatial_anchor && rhs.0.msft_spatial_anchor; diff --git a/src/input.rs b/src/input.rs index 33b1785..561969f 100644 --- a/src/input.rs +++ b/src/input.rs @@ -50,8 +50,14 @@ impl XrInput { // xr::Posef::IDENTITY, // )?; - let stage = - session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?; + let stage = match instance.exts().ext_local_floor { + None => session + .create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?, + Some(_) => session.create_reference_space( + xr::ReferenceSpaceType::LOCAL_FLOOR_EXT, + xr::Posef::IDENTITY, + )?, + }; let head = session.create_reference_space(xr::ReferenceSpaceType::VIEW, xr::Posef::IDENTITY)?; // let y = stage diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 9964ce6..9e71511 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -11,6 +11,8 @@ use crate::{ use super::oculus_touch::ActionSets; +pub use xr::sys::NULL_PATH; + pub struct OpenXrActionsPlugin; impl Plugin for OpenXrActionsPlugin { fn build(&self, app: &mut App) { From aae751bdabcecc929bf42ab2ec9c9bd20baafdfb Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 9 Jan 2024 20:34:10 +0100 Subject: [PATCH 10/80] changed vulkan init code --- Cargo.toml | 5 +- examples/demo/src/lib.rs | 4 - src/graphics/mod.rs | 13 ++-- src/graphics/vulkan.rs | 107 +++++++++++++++++---------- src/input.rs | 4 +- src/lib.rs | 39 +++++----- src/resource_macros.rs | 55 ++++++++++---- src/resources.rs | 21 +++++- src/xr_init.rs | 19 +++-- src/xr_input/actions.rs | 2 +- src/xr_input/debug_gizmos.rs | 2 +- src/xr_input/hands/hand_tracking.rs | 2 +- src/xr_input/oculus_touch.rs | 2 +- src/xr_input/prototype_locomotion.rs | 20 ++--- src/xr_input/trackers.rs | 2 - src/xr_input/xr_camera.rs | 14 ++-- 16 files changed, 191 insertions(+), 120 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2836170..2f6c6f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ force-link = ["openxr/linked"] members = ["examples/android", "examples/demo"] [dependencies] -anyhow = "1.0.75" +# anyhow = "1.0.75" ash = "0.37.3" bevy = "0.12" futures-lite = "2.0.1" @@ -23,6 +23,7 @@ mint = "0.5.9" wgpu = "0.17.1" wgpu-core = { version = "0.17.1", features = ["vulkan"] } wgpu-hal = "0.17.1" +eyre = "0.6.11" [target.'cfg(windows)'.dependencies] openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = [ @@ -42,8 +43,8 @@ openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features [dev-dependencies] bevy = "0.12" -color-eyre = "0.6.2" bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } +color-eyre = "0.6.2" [[example]] name = "xr" diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index 22529c7..ad92cd8 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -560,8 +560,6 @@ fn request_cube_spawn( ) { timer.0.tick(time.delta()); if timer.0.finished() { - //lock frame - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get controller triggers @@ -628,8 +626,6 @@ fn prototype_interaction_input( >, action_sets: Res, ) { - //lock frame - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get controller triggers diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 0aa3dfc..49baa88 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -7,8 +7,8 @@ use wgpu::Instance; use crate::input::XrInput; use crate::resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, + XrSessionRunning, XrSwapchain, XrViews, XrFrameWaiter, }; use openxr as xr; @@ -44,7 +44,7 @@ pub fn initialize_xr_graphics( reqeusted_extensions: XrExtensions, prefered_blend_mode: XrPreferdBlendMode, app_info: XrAppInfo, -) -> anyhow::Result<( +) -> eyre::Result<( RenderDevice, RenderQueue, RenderAdapterInfo, @@ -62,13 +62,14 @@ pub fn initialize_xr_graphics( XrViews, XrFrameState, )> { - vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info) + // vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info) + todo!() } -pub fn xr_entry() -> anyhow::Result { +pub fn xr_entry() -> eyre::Result { #[cfg(windows)] let entry = Ok(xr::Entry::linked()); #[cfg(not(windows))] - let entry = unsafe { xr::Entry::load().map_err(|e| anyhow::anyhow!(e)) }; + let entry = unsafe { xr::Entry::load().map_err(|e| eyre::eyre!(e)) }; entry } diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index ba9814e..1411491 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -1,52 +1,45 @@ use std::ffi::{c_void, CString}; use std::sync::atomic::AtomicBool; -use std::sync::{Arc, Mutex}; +use std::sync::Mutex; -use anyhow::Context; +// use anyhow::Context; use ash::vk::{self, Handle}; use bevy::math::uvec2; use bevy::prelude::*; use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}; use bevy::window::RawHandleWrapper; +use eyre::{Context, ContextCompat}; use openxr as xr; use wgpu::Instance; +use wgpu_hal::{api::Vulkan as V, Api}; use xr::EnvironmentBlendMode; use crate::graphics::extensions::XrExtensions; use crate::input::XrInput; use crate::resources::{ - Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, - XrInstance, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, + OXrSessionSetupInfo, Swapchain, SwapchainInner, VulkanOXrSessionSetupInfo, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, + XrSession, XrSessionRunning, XrSwapchain, XrViews, }; use crate::VIEW_TYPE; use super::{XrAppInfo, XrPreferdBlendMode}; -pub fn initialize_xr_graphics( +pub fn initialize_xr_instance( window: Option, reqeusted_extensions: XrExtensions, prefered_blend_mode: XrPreferdBlendMode, app_info: XrAppInfo, -) -> anyhow::Result<( +) -> eyre::Result<( + XrInstance, + OXrSessionSetupInfo, + XrEnvironmentBlendMode, RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter, Instance, - XrInstance, - XrSession, - XrEnvironmentBlendMode, - XrResolution, - XrFormat, - XrSessionRunning, - XrFrameWaiter, - XrSwapchain, - XrInput, - XrViews, - XrFrameState, )> { - use wgpu_hal::{api::Vulkan as V, Api}; - let xr_entry = super::xr_entry()?; #[cfg(target_os = "android")] @@ -108,7 +101,6 @@ pub fn initialize_xr_graphics( _ => EnvironmentBlendMode::OPAQUE, }; - #[cfg(not(target_os = "android"))] let vk_target_version = vk::make_api_version(0, 1, 2, 0); #[cfg(not(target_os = "android"))] @@ -296,22 +288,66 @@ pub fn initialize_xr_graphics( None, ) }?; + Ok(( + xr_instance.into(), + OXrSessionSetupInfo::Vulkan(VulkanOXrSessionSetupInfo { + device_ptr: vk_device_ptr, + physical_device_ptr: vk_physical_device_ptr, + vk_instance_ptr, + queue_family_index, + xr_system_id, + }), + blend_mode.into(), + wgpu_device.into(), + RenderQueue(wgpu_queue.into()), + RenderAdapterInfo(wgpu_adapter.get_info()), + RenderAdapter(wgpu_adapter.into()), + wgpu_instance.into(), + )) +} +pub fn initialize_xr_graphics( + window: Option, + ptrs: &OXrSessionSetupInfo, + xr_instance: &XrInstance, + render_device: &RenderDevice, + render_adapter: &RenderAdapter, + wgpu_instance: &Instance, +) -> eyre::Result<( + XrInstance, + XrSession, + XrResolution, + XrFormat, + XrSessionRunning, + XrFrameWaiter, + XrSwapchain, + XrInput, + XrViews, + XrFrameState, +)> { + let wgpu_device = render_device.wgpu_device(); + let wgpu_adapter = &render_adapter.0; + + #[allow(unreachable_patterns)] + let setup_info = match ptrs { + OXrSessionSetupInfo::Vulkan(v) => v, + _ => eyre::bail!("Wrong Graphics Api"), + }; let (session, frame_wait, frame_stream) = unsafe { xr_instance.create_session::( - xr_system_id, + xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY)?, &xr::vulkan::SessionCreateInfo { - instance: vk_instance_ptr, - physical_device: vk_physical_device_ptr, - device: vk_device_ptr, - queue_family_index, + instance: setup_info.vk_instance_ptr, + physical_device: setup_info.physical_device_ptr, + device: setup_info.device_ptr, + queue_family_index: setup_info.queue_family_index, queue_index: 0, }, ) }?; - let views = xr_instance.enumerate_view_configuration_views(xr_system_id, VIEW_TYPE)?; - + let views = + xr_instance.enumerate_view_configuration_views(setup_info.xr_system_id, VIEW_TYPE)?; let surface = window.map(|wrapper| unsafe { // SAFETY: Plugins should be set up on the main thread. let handle = wrapper.get_handle(); @@ -346,6 +382,7 @@ pub fn initialize_xr_graphics( mip_count: 1, }) .unwrap(); + let images = handle.enumerate_images().unwrap(); let buffers = images @@ -399,18 +436,12 @@ pub fn initialize_xr_graphics( .collect(); Ok(( - wgpu_device.into(), - RenderQueue(Arc::new(wgpu_queue)), - RenderAdapterInfo(wgpu_adapter.get_info()), - RenderAdapter(Arc::new(wgpu_adapter)), - wgpu_instance, xr_instance.clone().into(), session.clone().into_any_graphics().into(), - blend_mode.into(), resolution.into(), swapchain_format.into(), AtomicBool::new(false).into(), - Mutex::new(frame_wait).into(), + frame_wait.into(), Swapchain::Vulkan(SwapchainInner { stream: Mutex::new(frame_stream), handle: Mutex::new(handle), @@ -418,13 +449,13 @@ pub fn initialize_xr_graphics( image_index: Mutex::new(0), }) .into(), - XrInput::new(xr_instance, session.into_any_graphics())?, - Mutex::default().into(), - Mutex::new(xr::FrameState { + XrInput::new(xr_instance, &session.into_any_graphics())?, + Vec::default().into(), + xr::FrameState { predicted_display_time: xr::Time::from_nanos(1), predicted_display_period: xr::Duration::from_nanos(1), should_render: true, - }) + } .into(), )) } diff --git a/src/input.rs b/src/input.rs index 561969f..cf81c0e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -16,8 +16,8 @@ pub struct XrInput { impl XrInput { pub fn new( - instance: xr::Instance, - session: xr::Session, + instance: &xr::Instance, + session: &xr::Session, // frame_state: &FrameState, ) -> xr::Result { // let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 9a77e87..6f9a591 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,7 +26,7 @@ use input::XrInput; use openxr as xr; // use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; use resources::*; -use xr::FormFactor; +use xr::{FormFactor, FrameWaiter}; use xr_init::{xr_only, XrEnableStatus, XrRenderData}; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; @@ -57,7 +57,7 @@ pub struct FutureXrResources( XrResolution, XrFormat, XrSessionRunning, - XrFrameWaiter, + // XrFrameWaiter, XrSwapchain, XrInput, XrViews, @@ -110,7 +110,7 @@ impl Plugin for OpenXrPlugin { app.insert_resource(resolution.clone()); app.insert_resource(format.clone()); app.insert_resource(session_running.clone()); - app.insert_resource(frame_waiter.clone()); + app.insert_resource(frame_waiter); app.insert_resource(swapchain.clone()); app.insert_resource(input.clone()); app.insert_resource(views.clone()); @@ -122,7 +122,6 @@ impl Plugin for OpenXrPlugin { xr_resolution: resolution, xr_format: format, xr_session_running: session_running, - xr_frame_waiter: frame_waiter, xr_swapchain: swapchain, xr_input: input, xr_views: views, @@ -221,7 +220,7 @@ impl Plugin for OpenXrPlugin { render_app.insert_resource(data.xr_resolution.clone()); render_app.insert_resource(data.xr_format.clone()); render_app.insert_resource(data.xr_session_running.clone()); - render_app.insert_resource(data.xr_frame_waiter.clone()); + // render_app.insert_resource(data.xr_frame_waiter.clone()); render_app.insert_resource(data.xr_swapchain.clone()); render_app.insert_resource(data.xr_input.clone()); render_app.insert_resource(data.xr_views.clone()); @@ -286,10 +285,10 @@ pub fn xr_begin_frame( instance: Res, session: Res, session_running: Res, - frame_state: Res, - frame_waiter: Res, + mut frame_state: ResMut, + mut frame_waiter: ResMut, swapchain: Res, - views: Res, + mut views: ResMut, input: Res, mut app_exit: EventWriter, ) { @@ -303,6 +302,7 @@ pub fn xr_begin_frame( // find quit messages! info!("entered XR state {:?}", e.state()); match e.state() { + // xr_frame_waiter: frame_waiter, xr::SessionState::READY => { session.begin(VIEW_TYPE).unwrap(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); @@ -316,6 +316,7 @@ pub fn xr_begin_frame( app_exit.send(AppExit); return; } + _ => {} } } @@ -329,8 +330,8 @@ pub fn xr_begin_frame( } { let _span = info_span!("xr_wait_frame").entered(); - *frame_state.lock().unwrap() = match frame_waiter.lock().unwrap().wait() { - Ok(a) => a, + *frame_state = match frame_waiter.wait() { + Ok(a) => a.into(), Err(e) => { warn!("error: {}", e); return; @@ -343,12 +344,8 @@ pub fn xr_begin_frame( } { let _span = info_span!("xr_locate_views").entered(); - *views.lock().unwrap() = session - .locate_views( - VIEW_TYPE, - frame_state.lock().unwrap().predicted_display_time, - &input.stage, - ) + **views = session + .locate_views(VIEW_TYPE, frame_state.predicted_display_time, &input.stage) .unwrap() .1; } @@ -402,8 +399,8 @@ pub fn end_frame( { let _span = info_span!("xr_end_frame").entered(); let result = swapchain.end( - xr_frame_state.lock().unwrap().predicted_display_time, - &views.lock().unwrap(), + xr_frame_state.predicted_display_time, + &views, &input.stage, **resolution, **environment_blend_mode, @@ -417,15 +414,15 @@ pub fn end_frame( } pub fn locate_views( - views: Res, + mut views: ResMut, input: Res, session: Res, xr_frame_state: Res, ) { let _span = info_span!("xr_locate_views").entered(); - *views.lock().unwrap() = match session.locate_views( + **views = match session.locate_views( VIEW_TYPE, - xr_frame_state.lock().unwrap().predicted_display_time, + xr_frame_state.predicted_display_time, &input.stage, ) { Ok(this) => this, diff --git a/src/resource_macros.rs b/src/resource_macros.rs index 5f4df4e..933ce20 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -1,7 +1,7 @@ #[macro_export] macro_rules! xr_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource)] + #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] pub struct $wrapper_type($xr_type); impl $wrapper_type { @@ -10,13 +10,13 @@ macro_rules! xr_resource_wrapper { } } - impl std::ops::Deref for $wrapper_type { - type Target = $xr_type; - - fn deref(&self) -> &Self::Target { - &self.0 - } - } + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // &self.0 + // } + // } impl From<$xr_type> for $wrapper_type { fn from(value: $xr_type) -> Self { @@ -29,7 +29,7 @@ macro_rules! xr_resource_wrapper { #[macro_export] macro_rules! xr_arc_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource)] + #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] pub struct $wrapper_type(std::sync::Arc<$xr_type>); impl $wrapper_type { @@ -38,13 +38,41 @@ macro_rules! xr_arc_resource_wrapper { } } - impl std::ops::Deref for $wrapper_type { - type Target = $xr_type; + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // self.0.as_ref() + // } + // } - fn deref(&self) -> &Self::Target { - self.0.as_ref() + impl From<$xr_type> for $wrapper_type { + fn from(value: $xr_type) -> Self { + Self::new(value) } } + }; +} + +#[macro_export] +macro_rules! xr_no_clone_resource_wrapper { + ($wrapper_type:ident, $xr_type:ty) => { + #[derive(bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] + pub struct $wrapper_type($xr_type); + + impl $wrapper_type { + pub fn new(value: $xr_type) -> Self { + Self(value) + } + } + + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // &self.0 + // } + // } impl From<$xr_type> for $wrapper_type { fn from(value: $xr_type) -> Self { @@ -55,4 +83,5 @@ macro_rules! xr_arc_resource_wrapper { } pub use xr_arc_resource_wrapper; +pub use xr_no_clone_resource_wrapper; pub use xr_resource_wrapper; diff --git a/src/resources.rs b/src/resources.rs index a798955..62f2855 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -1,3 +1,4 @@ +use std::ffi::c_void; use std::sync::atomic::AtomicBool; use std::sync::Mutex; @@ -5,17 +6,31 @@ use std::sync::Mutex; use crate::resource_macros::*; use bevy::prelude::*; use openxr as xr; +use xr::ViewConfigurationView; xr_resource_wrapper!(XrInstance, xr::Instance); xr_resource_wrapper!(XrSession, xr::Session); xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); xr_resource_wrapper!(XrResolution, UVec2); xr_resource_wrapper!(XrFormat, wgpu::TextureFormat); +xr_resource_wrapper!(XrFrameState, xr::FrameState); +xr_resource_wrapper!(XrViews, Vec); xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool); -xr_arc_resource_wrapper!(XrFrameWaiter, Mutex); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); -xr_arc_resource_wrapper!(XrFrameState, Mutex); -xr_arc_resource_wrapper!(XrViews, Mutex>); +xr_no_clone_resource_wrapper!(XrFrameWaiter, xr::FrameWaiter); + +pub(crate) struct VulkanOXrSessionSetupInfo { + pub(crate) device_ptr: *const c_void, + pub(crate) physical_device_ptr: *const c_void, + pub(crate) vk_instance_ptr: *const c_void, + pub(crate) queue_family_index: u32, + pub(crate) xr_system_id: xr::SystemId, + // pub(crate) swapchain_create_info: xr::SwapchainCreateInfo +} + +pub(crate) enum OXrSessionSetupInfo { + Vulkan(VulkanOXrSessionSetupInfo), +} pub enum Swapchain { Vulkan(SwapchainInner), diff --git a/src/xr_init.rs b/src/xr_init.rs index 499e7ce..f60b0ba 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -11,16 +11,16 @@ use bevy::{ renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}, settings::WgpuSettings, }, - window::RawHandleWrapper, + window::{RawHandleWrapper, PrimaryWindow}, }; use wgpu::Instance; use crate::{ input::XrInput, resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, - }, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, + XrSessionRunning, XrSwapchain, XrViews, + }, graphics, }; #[derive(Resource, Clone)] @@ -40,7 +40,7 @@ pub struct XrRenderData { pub xr_resolution: XrResolution, pub xr_format: XrFormat, pub xr_session_running: XrSessionRunning, - pub xr_frame_waiter: XrFrameWaiter, + // pub xr_frame_waiter: XrFrameWaiter, pub xr_swapchain: XrSwapchain, pub xr_input: XrInput, pub xr_views: XrViews, @@ -174,6 +174,14 @@ pub fn update_xr_stuff(world: &mut World) { world.run_schedule(XrPostRenderUpdate); } +fn setup_xr_graphics() { + +} + +fn enable_xr( + +) {} + // fn handle_xr_enable_requests( // primary_window: Query<&RawHandleWrapper, With>, // mut commands: Commands, @@ -183,7 +191,6 @@ pub fn update_xr_stuff(world: &mut World) { // // Just to force this system onto the main thread because of unsafe code // let _ = on_main; // -// commands.insert_resource(XrEnableStatus::Waiting); // let (creation_data, xr_data) = match next_state.into_inner() { // XrNextEnabledState::Enabled => { // let ( diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 9e71511..748968d 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -206,7 +206,7 @@ impl SetupActionSet { for binding in bindings { self.actions .get_mut(binding.action) - .ok_or(anyhow::anyhow!("Missing Action: {}", binding.action)) + .ok_or(eyre::eyre!("Missing Action: {}", binding.action)) .unwrap() .bindings .entry(device_path) diff --git a/src/xr_input/debug_gizmos.rs b/src/xr_input/debug_gizmos.rs index c67d900..ff93ef8 100644 --- a/src/xr_input/debug_gizmos.rs +++ b/src/xr_input/debug_gizmos.rs @@ -91,7 +91,7 @@ pub fn draw_gizmos( // } // } //lock frame - let frame_state = *frame_state.lock().unwrap(); + // let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); let root = tracking_root_query.get_single(); diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index 1af55df..fd8c920 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -87,7 +87,7 @@ impl<'a> HandTrackingRef<'a> { Hand::Left => &self.tracking.left_hand, Hand::Right => &self.tracking.right_hand, }, - self.frame_state.lock().unwrap().predicted_display_time, + self.frame_state.predicted_display_time, ) .unwrap() .map(|joints| { diff --git a/src/xr_input/oculus_touch.rs b/src/xr_input/oculus_touch.rs index 6f405f8..27111c4 100644 --- a/src/xr_input/oculus_touch.rs +++ b/src/xr_input/oculus_touch.rs @@ -371,7 +371,7 @@ pub struct OculusController { pub aim_space: Option>, } impl OculusController { - pub fn new(mut action_sets: ResMut) -> anyhow::Result { + pub fn new(mut action_sets: ResMut) -> eyre::Result { let action_set = action_sets.add_action_set("oculus_input", "Oculus Touch Controller Input".into(), 0); action_set.new_action( diff --git a/src/xr_input/prototype_locomotion.rs b/src/xr_input/prototype_locomotion.rs index 52bf355..791ee04 100644 --- a/src/xr_input/prototype_locomotion.rs +++ b/src/xr_input/prototype_locomotion.rs @@ -70,17 +70,13 @@ pub fn proto_locomotion( config_option: Option>, action_sets: Res, ) { - match config_option { - Some(_) => (), + let mut config = match config_option { + Some(c) => c, None => { info!("no locomotion config"); return; } - } - //i hate this but im too tired to think - let mut config = config_option.unwrap(); - //lock frame - let frame_state = *frame_state.lock().unwrap(); + }; //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); let root = tracking_root_query.get_single_mut(); @@ -92,8 +88,7 @@ pub fn proto_locomotion( let reference_quat; match config.locomotion_type { LocomotionType::Head => { - let v = views.lock().unwrap(); - let views = v.get(0); + let views = views.first(); match views { Some(view) => { reference_quat = view.pose.orientation.to_quat(); @@ -127,8 +122,7 @@ pub fn proto_locomotion( rot_input * config.smooth_rotation_speed * time.delta_seconds(), ); //apply rotation - let v = views.lock().unwrap(); - let views = v.get(0); + let views = views.first(); match views { Some(view) => { let mut hmd_translation = view.pose.position.to_vec3(); @@ -159,8 +153,8 @@ pub fn proto_locomotion( let smoth_rot = Quat::from_axis_angle(position.0.up(), config.snap_angle * dir); //apply rotation - let v = views.lock().unwrap(); - let views = v.get(0); + let v = views; + let views = v.first(); match views { Some(view) => { let mut hmd_translation = view.pose.position.to_vec3(); diff --git a/src/xr_input/trackers.rs b/src/xr_input/trackers.rs index a57e8a8..eea28c1 100644 --- a/src/xr_input/trackers.rs +++ b/src/xr_input/trackers.rs @@ -66,8 +66,6 @@ pub fn update_open_xr_controllers( session: Res, action_sets: Res, ) { - //lock dat frame? - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get left controller diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 64528f8..210feaa 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -241,22 +241,24 @@ impl CameraProjection for XRProjection { pub fn xr_camera_head_sync( views: ResMut, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, ) { - let mut f = || -> Option<()> { + fn f( + views: ResMut, + mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + ) -> Option<()> { //TODO calculate HMD position for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { let view_idx = match camera_type { XrCameraType::Xr(eye) => *eye as usize, XrCameraType::Flatscreen => return None, }; - let v = views.lock().unwrap(); - let view = v.get(view_idx)?; + let view = views.get(view_idx)?; xr_projection.fov = view.fov; transform.rotation = view.pose.orientation.to_quat(); transform.translation = view.pose.position.to_vec3(); } Some(()) - }; - let _ = f(); + } + let _ = f(views, query); } From cbf9485bb78456f83d935f5b8924941fd3913648 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Sun, 14 Jan 2024 00:15:14 +0100 Subject: [PATCH 11/80] working --- examples/demo/src/lib.rs | 1 - examples/globe.rs | 3 +- examples/xr.rs | 2 - src/graphics/mod.rs | 133 +++++++++++++++++++++++++++----- src/graphics/vulkan.rs | 4 +- src/input.rs | 4 +- src/lib.rs | 158 ++++++++++++--------------------------- src/resource_macros.rs | 16 +++- src/resources.rs | 20 ++++- src/xr_init.rs | 38 +++++----- src/xr_input/trackers.rs | 9 ++- 11 files changed, 224 insertions(+), 164 deletions(-) diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index ad92cd8..2867a23 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -50,7 +50,6 @@ fn input_stuff( match status.into_inner() { XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable), XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable), - XrEnableStatus::Waiting => (), } } } diff --git a/examples/globe.rs b/examples/globe.rs index cf58bea..192916b 100644 --- a/examples/globe.rs +++ b/examples/globe.rs @@ -172,8 +172,7 @@ fn pull_to_ground( let (globe_pos, globe) = globe.single(); // Get player position (position of playground + position within playground) - let v = views.lock().unwrap(); - let Some(view) = v.get(0) else { return }; + let Some(view) = views.first() else { return }; let mut hmd_translation = view.pose.position.to_vec3(); hmd_translation.y = 0.0; let local = root.translation; diff --git a/examples/xr.rs b/examples/xr.rs index b38d79a..6f8d2de 100644 --- a/examples/xr.rs +++ b/examples/xr.rs @@ -167,8 +167,6 @@ fn prototype_interaction_input( >, action_sets: Res, ) { - //lock frame - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get controller triggers diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 49baa88..24e80ff 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -1,14 +1,19 @@ pub mod extensions; mod vulkan; -use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}; -use bevy::window::RawHandleWrapper; +use bevy::ecs::query::With; +use bevy::ecs::system::{Query, SystemState}; +use bevy::ecs::world::World; +use bevy::render::renderer::{ + RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, +}; +use bevy::window::{PrimaryWindow, RawHandleWrapper}; use wgpu::Instance; use crate::input::XrInput; use crate::resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, - XrSessionRunning, XrSwapchain, XrViews, XrFrameWaiter, + OXrSessionSetupInfo, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, + XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }; use openxr as xr; @@ -39,20 +44,15 @@ impl Default for XrAppInfo { } } -pub fn initialize_xr_graphics( +pub fn start_xr_session( window: Option, - reqeusted_extensions: XrExtensions, - prefered_blend_mode: XrPreferdBlendMode, - app_info: XrAppInfo, + session_setup_data: &OXrSessionSetupInfo, + xr_instance: &XrInstance, + render_device: &RenderDevice, + render_adapter: &RenderAdapter, + wgpu_instance: &Instance, ) -> eyre::Result<( - RenderDevice, - RenderQueue, - RenderAdapterInfo, - RenderAdapter, - Instance, - XrInstance, XrSession, - XrEnvironmentBlendMode, XrResolution, XrFormat, XrSessionRunning, @@ -62,8 +62,107 @@ pub fn initialize_xr_graphics( XrViews, XrFrameState, )> { - // vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info) - todo!() + vulkan::start_xr_session( + window, + session_setup_data, + xr_instance, + render_device, + render_adapter, + wgpu_instance, + ) +} +pub fn initialize_xr_instance( + window: Option, + reqeusted_extensions: XrExtensions, + prefered_blend_mode: XrPreferdBlendMode, + app_info: XrAppInfo, +) -> eyre::Result<( + XrInstance, + OXrSessionSetupInfo, + XrEnvironmentBlendMode, + RenderDevice, + RenderQueue, + RenderAdapterInfo, + RenderAdapter, + Instance, +)> { + vulkan::initialize_xr_instance(window, reqeusted_extensions, prefered_blend_mode, app_info) +} + +pub fn try_full_init( + world: &mut World, + reqeusted_extensions: XrExtensions, + prefered_blend_mode: XrPreferdBlendMode, + app_info: XrAppInfo, +) -> eyre::Result<( + RenderDevice, + RenderQueue, + RenderAdapterInfo, + RenderAdapter, + RenderInstance, +)> { + let mut system_state: SystemState>> = + SystemState::new(world); + let primary_window = system_state.get(&world).get_single().ok().cloned(); + let ( + xr_instance, + setup_info, + blend_mode, + render_device, + render_queue, + render_adapter_info, + render_adapter, + wgpu_instance, + ) = initialize_xr_instance( + primary_window.clone(), + reqeusted_extensions, + prefered_blend_mode, + app_info, + )?; + world.insert_resource(xr_instance); + world.insert_non_send_resource(setup_info); + // TODO: move BlendMode the session init? + world.insert_resource(blend_mode); + let setup_info = world + .get_non_send_resource::() + .unwrap(); + let xr_instance = world.get_resource::().unwrap(); + + let ( + xr_session, + xr_resolution, + xr_format, + xr_session_running, + xr_frame_waiter, + xr_swapchain, + xr_input, + xr_views, + xr_frame_state, + ) = start_xr_session( + primary_window, + setup_info, + xr_instance, + &render_device, + &render_adapter, + &wgpu_instance, + )?; + world.insert_resource(xr_session); + world.insert_resource(xr_resolution); + world.insert_resource(xr_format); + world.insert_resource(xr_session_running); + world.insert_resource(xr_frame_waiter); + world.insert_resource(xr_swapchain); + world.insert_resource(xr_input); + world.insert_resource(xr_views); + world.insert_resource(xr_frame_state); + + Ok(( + render_device, + render_queue, + render_adapter_info, + render_adapter, + RenderInstance(wgpu_instance.into()), + )) } pub fn xr_entry() -> eyre::Result { diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index 1411491..ac0948d 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -306,7 +306,7 @@ pub fn initialize_xr_instance( )) } -pub fn initialize_xr_graphics( +pub fn start_xr_session( window: Option, ptrs: &OXrSessionSetupInfo, xr_instance: &XrInstance, @@ -314,7 +314,6 @@ pub fn initialize_xr_graphics( render_adapter: &RenderAdapter, wgpu_instance: &Instance, ) -> eyre::Result<( - XrInstance, XrSession, XrResolution, XrFormat, @@ -436,7 +435,6 @@ pub fn initialize_xr_graphics( .collect(); Ok(( - xr_instance.clone().into(), session.clone().into_any_graphics().into(), resolution.into(), swapchain_format.into(), diff --git a/src/input.rs b/src/input.rs index cf81c0e..8a98a7e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1,10 +1,10 @@ use std::sync::Arc; -use bevy::prelude::*; +use bevy::{prelude::*, render::extract_resource::ExtractResource}; use openxr as xr; use xr::{FrameState, FrameWaiter, ViewConfigurationType}; -#[derive(Clone, Resource)] +#[derive(Clone, Resource, ExtractResource)] pub struct XrInput { //pub action_set: xr::ActionSet, //pub hand_pose: xr::Action, diff --git a/src/lib.rs b/src/lib.rs index 6f9a591..ea1e496 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,11 +15,13 @@ use bevy::app::{AppExit, PluginGroupBuilder}; use bevy::ecs::system::SystemState; use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; +use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; +use eyre::anyhow; use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; use input::XrInput; @@ -57,7 +59,6 @@ pub struct FutureXrResources( XrResolution, XrFormat, XrSessionRunning, - // XrFrameWaiter, XrSwapchain, XrInput, XrViews, @@ -66,68 +67,20 @@ pub struct FutureXrResources( >, >, ); -// fn mr_test(mut commands: Commands, passthrough_layer: Option>) { -// commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); -// } impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { - let mut system_state: SystemState>> = - SystemState::new(&mut app.world); - let primary_window = system_state.get(&app.world).get_single().ok().cloned(); - #[cfg(not(target_arch = "wasm32"))] - match graphics::initialize_xr_graphics( - primary_window.clone(), + match graphics::try_full_init( + &mut app.world, self.reqeusted_extensions.clone(), self.prefered_blend_mode, self.app_info.clone(), ) { - Ok(( - device, - queue, - adapter_info, - render_adapter, - instance, - xr_instance, - session, - blend_mode, - resolution, - format, - session_running, - frame_waiter, - swapchain, - input, - views, - frame_state, - )) => { - // std::thread::sleep(Duration::from_secs(5)); + Ok((device, queue, adapter_info, render_adapter, instance)) => { debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); debug!("Configured wgpu adapter Features: {:#?}", device.features()); - app.insert_resource(xr_instance.clone()); - app.insert_resource(session.clone()); - app.insert_resource(blend_mode.clone()); - app.insert_resource(resolution.clone()); - app.insert_resource(format.clone()); - app.insert_resource(session_running.clone()); - app.insert_resource(frame_waiter); - app.insert_resource(swapchain.clone()); - app.insert_resource(input.clone()); - app.insert_resource(views.clone()); - app.insert_resource(frame_state.clone()); - let xr_data = XrRenderData { - xr_instance, - xr_session: session, - xr_blend_mode: blend_mode, - xr_resolution: resolution, - xr_format: format, - xr_session_running: session_running, - xr_swapchain: swapchain, - xr_input: input, - xr_views: views, - xr_frame_state: frame_state, - }; - app.insert_resource(xr_data); + warn!("Starting in Xr"); app.insert_resource(ActionSets(vec![])); app.add_plugins(RenderPlugin { render_creation: RenderCreation::Manual( @@ -135,14 +88,16 @@ impl Plugin for OpenXrPlugin { queue, adapter_info, render_adapter, - RenderInstance(Arc::new(instance)), + instance, ), }); + app.add_plugins(ExtractResourcePlugin::::default()); app.insert_resource(XrEnableStatus::Enabled); } Err(err) => { warn!("OpenXR Failed to initialize: {}", err); app.add_plugins(RenderPlugin::default()); + app.add_plugins(ExtractResourcePlugin::::default()); app.insert_resource(XrEnableStatus::Disabled); } } @@ -154,58 +109,48 @@ impl Plugin for OpenXrPlugin { } } - fn ready(&self, app: &App) -> bool { - app.world - .get_resource::() - .map(|frr| *frr != XrEnableStatus::Waiting) - .unwrap_or(true) - } - fn finish(&self, app: &mut App) { // TODO: Split this up into the indevidual resources - if let Some(data) = app.world.get_resource::().cloned() { - let hands = data.xr_instance.exts().ext_hand_tracking.is_some() - && data - .xr_instance + if app.world.get_resource::() == Some(&XrEnableStatus::Enabled) { + warn!("finished xr init"); + let xr_instance = app + .world + .get_resource::() + .expect("should exist"); + let xr_session = app.world.get_resource::().expect("should exist"); + let hands = xr_instance.exts().ext_hand_tracking.is_some() + && xr_instance .supports_hand_tracking( - data.xr_instance + xr_instance .system(FormFactor::HEAD_MOUNTED_DISPLAY) .unwrap(), ) .is_ok_and(|v| v); if hands { - app.insert_resource(HandTrackingData::new(&data.xr_session).unwrap()); + app.insert_resource(HandTrackingData::new(xr_session).unwrap()); } else { app.insert_resource(DisableHandTracking::Both); } - // let passthrough = data.xr_instance.exts().fb_passthrough.is_some() - // && supports_passthrough( - // &data.xr_instance, - // data.xr_instance - // .system(FormFactor::HEAD_MOUNTED_DISPLAY) - // .unwrap(), - // ) - // .is_ok_and(|v| v); - // if passthrough { - // info!("Passthrough!"); - // let (pl, p) = start_passthrough(&data); - // app.insert_resource(pl); - // app.insert_resource(p); - // // if !app.world.contains_resource::() { - // // info!("ClearColor!"); - // // } - // } + let xr_swapchain = app + .world + .get_resource::() + .expect("should exist"); + let xr_resolution = app + .world + .get_resource::() + .expect("should exist"); + let xr_format = app.world.get_resource::().expect("should exist"); - let (left, right) = data.xr_swapchain.get_render_views(); + let (left, right) = xr_swapchain.get_render_views(); let left = ManualTextureView { texture_view: left.into(), - size: *data.xr_resolution, - format: *data.xr_format, + size: **xr_resolution, + format: **xr_format, }; let right = ManualTextureView { texture_view: right.into(), - size: *data.xr_resolution, - format: *data.xr_format, + size: **xr_resolution, + format: **xr_format, }; app.add_systems(PreUpdate, xr_begin_frame.run_if(xr_only())); let mut manual_texture_views = app.world.resource_mut::(); @@ -213,19 +158,6 @@ impl Plugin for OpenXrPlugin { manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); drop(manual_texture_views); let render_app = app.sub_app_mut(RenderApp); - - render_app.insert_resource(data.xr_instance.clone()); - render_app.insert_resource(data.xr_session.clone()); - render_app.insert_resource(data.xr_blend_mode.clone()); - render_app.insert_resource(data.xr_resolution.clone()); - render_app.insert_resource(data.xr_format.clone()); - render_app.insert_resource(data.xr_session_running.clone()); - // render_app.insert_resource(data.xr_frame_waiter.clone()); - render_app.insert_resource(data.xr_swapchain.clone()); - render_app.insert_resource(data.xr_input.clone()); - render_app.insert_resource(data.xr_views.clone()); - render_app.insert_resource(data.xr_frame_state.clone()); - render_app.insert_resource(XrEnableStatus::Enabled); render_app.add_systems( Render, ( @@ -262,6 +194,8 @@ impl PluginGroup for DefaultXrPlugins { .add_before::(RenderRestartPlugin) .add(HandEmulationPlugin) .add(HandTrackingPlugin) + .add(XrResourcePlugin) + // .add(xr_init::RenderRestartPlugin) .set(WindowPlugin { #[cfg(not(target_os = "android"))] primary_window: Some(Window { @@ -302,7 +236,6 @@ pub fn xr_begin_frame( // find quit messages! info!("entered XR state {:?}", e.state()); match e.state() { - // xr_frame_waiter: frame_waiter, xr::SessionState::READY => { session.begin(VIEW_TYPE).unwrap(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); @@ -310,7 +243,6 @@ pub fn xr_begin_frame( xr::SessionState::STOPPING => { session.end().unwrap(); session_running.store(false, std::sync::atomic::Ordering::Relaxed); - app_exit.send(AppExit); } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { app_exit.send(AppExit); @@ -384,14 +316,20 @@ pub fn post_frame( } pub fn end_frame( - xr_frame_state: Res, - views: Res, - input: Res, - swapchain: Res, - resolution: Res, - environment_blend_mode: Res, + xr_frame_state: Option>, + views: Option>, + input: Option>, + swapchain: Option>, + resolution: Option>, + environment_blend_mode: Option>, // passthrough_layer: Option>, ) { + let xr_frame_state = xr_frame_state.unwrap(); + let views = views.unwrap(); + let input = input.unwrap(); + let swapchain = swapchain.unwrap(); + let resolution = resolution.unwrap(); + let environment_blend_mode = environment_blend_mode.unwrap(); { let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); diff --git a/src/resource_macros.rs b/src/resource_macros.rs index 933ce20..735ec50 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -1,7 +1,13 @@ #[macro_export] macro_rules! xr_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] + #[derive( + Clone, + bevy::prelude::Resource, + bevy::prelude::Deref, + bevy::prelude::DerefMut, + bevy::render::extract_resource::ExtractResource, + )] pub struct $wrapper_type($xr_type); impl $wrapper_type { @@ -29,7 +35,13 @@ macro_rules! xr_resource_wrapper { #[macro_export] macro_rules! xr_arc_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] + #[derive( + Clone, + bevy::prelude::Resource, + bevy::prelude::Deref, + bevy::prelude::DerefMut, + bevy::render::extract_resource::ExtractResource, + )] pub struct $wrapper_type(std::sync::Arc<$xr_type>); impl $wrapper_type { diff --git a/src/resources.rs b/src/resources.rs index 62f2855..b73cb1c 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -2,11 +2,13 @@ use std::ffi::c_void; use std::sync::atomic::AtomicBool; use std::sync::Mutex; +use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; use crate::resource_macros::*; +use crate::xr_init::XrEnableStatus; use bevy::prelude::*; +use bevy::render::extract_resource::ExtractResourcePlugin; use openxr as xr; -use xr::ViewConfigurationView; xr_resource_wrapper!(XrInstance, xr::Instance); xr_resource_wrapper!(XrSession, xr::Session); @@ -32,6 +34,20 @@ pub(crate) enum OXrSessionSetupInfo { Vulkan(VulkanOXrSessionSetupInfo), } +pub struct XrResourcePlugin; + +impl Plugin for XrResourcePlugin { + fn build(&self, app: &mut App) { + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + } +} + pub enum Swapchain { Vulkan(SwapchainInner), } @@ -153,7 +169,7 @@ impl SwapchainInner { }, }; let swapchain = self.handle.lock().unwrap(); - if views.len() == 0 { + if views.is_empty() { warn!("views are len of 0"); return Ok(()); } diff --git a/src/xr_init.rs b/src/xr_init.rs index f60b0ba..698835d 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -11,16 +11,17 @@ use bevy::{ renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}, settings::WgpuSettings, }, - window::{RawHandleWrapper, PrimaryWindow}, + window::{PrimaryWindow, RawHandleWrapper}, }; use wgpu::Instance; use crate::{ + graphics, input::XrInput, resources::{ XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, - }, graphics, + }, }; #[derive(Resource, Clone)] @@ -52,14 +53,13 @@ pub enum XrEnableRequest { TryEnable, TryDisable, } -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq)] +#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Reflect, ExtractResource)] pub enum XrEnableStatus { Enabled, Disabled, - Waiting, } -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Debug)] +#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Debug, ExtractResource)] pub enum XrNextEnabledState { Enabled, Disabled, @@ -67,29 +67,33 @@ pub enum XrNextEnabledState { pub struct RenderRestartPlugin; -#[derive(Resource)] -pub struct ForceMain; - #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPreSetup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrSetup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPrePostSetup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostSetup; #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPreCleanup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrCleanup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostCleanup; #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPreRenderUpdate; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrRenderUpdate; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostRenderUpdate; @@ -99,9 +103,9 @@ pub fn xr_only() -> impl FnMut(Option>) -> bool { impl Plugin for RenderRestartPlugin { fn build(&self, app: &mut App) { + info!("build RenderRestartPlugin"); add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()) - .insert_resource(ForceMain) .add_event::() .add_event::() .add_systems(PostStartup, setup_xr.run_if(xr_only())) @@ -174,13 +178,9 @@ pub fn update_xr_stuff(world: &mut World) { world.run_schedule(XrPostRenderUpdate); } -fn setup_xr_graphics() { +fn setup_xr_graphics() {} -} - -fn enable_xr( - -) {} +fn enable_xr() {} // fn handle_xr_enable_requests( // primary_window: Query<&RawHandleWrapper, With>, @@ -301,10 +301,10 @@ fn decide_next_xr_state( info!("Xr Already Disabled! ignoring request"); return; } - (_, Some(XrEnableStatus::Waiting)) => { - info!("Already Handling Request! ignoring request"); - return; - } + // (_, Some(XrEnableStatus::Waiting)) => { + // info!("Already Handling Request! ignoring request"); + // return; + // } _ => {} } let r = match request { diff --git a/src/xr_input/trackers.rs b/src/xr_input/trackers.rs index eea28c1..e607c16 100644 --- a/src/xr_input/trackers.rs +++ b/src/xr_input/trackers.rs @@ -1,3 +1,4 @@ +use bevy::hierarchy::Parent; use bevy::log::{debug, info}; use bevy::prelude::{ Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With, Without, @@ -30,17 +31,17 @@ pub struct OpenXRController; pub struct AimPose(pub Transform); pub fn adopt_open_xr_trackers( - query: Query>, + query: Query, Without)>, mut commands: Commands, - tracking_root_query: Query<(Entity, With)>, + tracking_root_query: Query>, ) { let root = tracking_root_query.get_single(); match root { - Ok(thing) => { + Ok(root) => { // info!("root is"); for tracker in query.iter() { info!("we got a new tracker"); - commands.entity(thing.0).add_child(tracker); + commands.entity(root).add_child(tracker); } } Err(_) => info!("root isnt spawned yet?"), From 144e786db940f99ca878e76965cd6539f23eda22 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Sun, 14 Jan 2024 02:12:49 +0100 Subject: [PATCH 12/80] ready to test --- src/lib.rs | 2 +- src/xr_init.rs | 79 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ea1e496..d1bd31a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -245,7 +245,7 @@ pub fn xr_begin_frame( session_running.store(false, std::sync::atomic::Ordering::Relaxed); } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { - app_exit.send(AppExit); + // app_exit.send(AppExit); return; } diff --git a/src/xr_init.rs b/src/xr_init.rs index 698835d..13ee973 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -8,7 +8,9 @@ use bevy::{ prelude::*, render::{ extract_resource::{ExtractResource, ExtractResourcePlugin}, - renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}, + renderer::{ + self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, + }, settings::WgpuSettings, }, window::{PrimaryWindow, RawHandleWrapper}, @@ -19,8 +21,8 @@ use crate::{ graphics, input::XrInput, resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, - XrSessionRunning, XrSwapchain, XrViews, + OXrSessionSetupInfo, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, + XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }, }; @@ -97,17 +99,21 @@ pub struct XrRenderUpdate; #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostRenderUpdate; -pub fn xr_only() -> impl FnMut(Option>) -> bool { - resource_exists_and_equals(XrEnableStatus::Enabled) +pub fn xr_only() -> impl FnMut(Option>, Res) -> bool { + |status, running| { + resource_exists_and_equals(XrEnableStatus::Enabled)(status) + && running.load(std::sync::atomic::Ordering::Relaxed) + } } impl Plugin for RenderRestartPlugin { fn build(&self, app: &mut App) { - info!("build RenderRestartPlugin"); add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()) .add_event::() .add_event::() + .add_event::() + .add_event::() .add_systems(PostStartup, setup_xr.run_if(xr_only())) .add_systems( PostUpdate, @@ -126,6 +132,8 @@ impl Plugin for RenderRestartPlugin { .chain(), ) .add_systems(XrCleanup, cleanup_oxr_session); + app.add_systems(PostUpdate, start_xr_session.run_if(on_event::())); + app.add_systems(PostUpdate, stop_xr_session.run_if(on_event::())); } } @@ -178,9 +186,64 @@ pub fn update_xr_stuff(world: &mut World) { world.run_schedule(XrPostRenderUpdate); } -fn setup_xr_graphics() {} +#[derive(Event, Clone, Copy, Default)] +pub struct StartXrSession; -fn enable_xr() {} +#[derive(Event, Clone, Copy, Default)] +pub struct EndXrSession; + +fn start_xr_session( + mut commands: Commands, + instance: Res, + primary_window: Query<&RawHandleWrapper, With>, + setup_info: NonSend, + render_device: Res, + render_adapter: Res, + render_instance: Res, +) { + let ( + xr_session, + xr_resolution, + xr_format, + xr_session_running, + xr_frame_waiter, + xr_swapchain, + xr_input, + xr_views, + xr_frame_state, + ) = match graphics::start_xr_session( + primary_window.get_single().cloned().ok(), + &setup_info, + &instance, + &render_device, + &render_adapter, + &render_instance, + ) { + Ok(data) => data, + Err(err) => { + error!("Unable to start OpenXR Session: {}", err); + return; + } + }; + commands.insert_resource(xr_session); + commands.insert_resource(xr_resolution); + commands.insert_resource(xr_format); + commands.insert_resource(xr_session_running); + commands.insert_resource(xr_frame_waiter); + commands.insert_resource(xr_swapchain); + commands.insert_resource(xr_input); + commands.insert_resource(xr_views); + commands.insert_resource(xr_frame_state); +} + +fn stop_xr_session(mut commands: Commands, session: ResMut) { + match session.request_exit() { + Ok(_) => {} + Err(err) => { + error!("Error while trying to request session exit: {}", err) + } + } +} // fn handle_xr_enable_requests( // primary_window: Query<&RawHandleWrapper, With>, From 87fe9093dfd14d4410d8cf5a0de038b407aac424 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 15 Jan 2024 05:08:49 +0100 Subject: [PATCH 13/80] fix android warning --- Cargo.toml | 4 ++++ src/lib.rs | 55 ++++++++++++++++++++++++++++++++++++------------ src/resources.rs | 8 +++++-- src/xr_init.rs | 21 +++++++++++++++--- 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2f6c6f5..bb93ce7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,10 @@ openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features "static", ] } +[target.'cfg(target_os = "android")'.dependencies] +ndk-context = "0.1" +jni = "0.20" + [dev-dependencies] bevy = "0.12" bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } diff --git a/src/lib.rs b/src/lib.rs index d1bd31a..4705e54 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod resources; pub mod xr_init; pub mod xr_input; +use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; use crate::xr_init::RenderRestartPlugin; @@ -70,6 +71,13 @@ pub struct FutureXrResources( impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { + app.insert_resource(XrSessionRunning::new(AtomicBool::new(false))); + // #[cfg(target_os = "android")] + // { + // let ctx = ndk_context::android_context(); + // let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); + // let env = vm.attach_current_thread_permanently(); + // } #[cfg(not(target_arch = "wasm32"))] match graphics::try_full_init( &mut app.world, @@ -91,8 +99,7 @@ impl Plugin for OpenXrPlugin { instance, ), }); - app.add_plugins(ExtractResourcePlugin::::default()); - app.insert_resource(XrEnableStatus::Enabled); + app.insert_resource(XrEnableStatus::Disabled); } Err(err) => { warn!("OpenXR Failed to initialize: {}", err); @@ -107,11 +114,17 @@ impl Plugin for OpenXrPlugin { app.add_plugins(RenderPlugin::default()); app.insert_resource(XrEnableStatus::Disabled); } + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_systems( + PreUpdate, + (xr_poll_events, xr_begin_frame.run_if(xr_only())).chain(), + ); } fn finish(&self, app: &mut App) { // TODO: Split this up into the indevidual resources - if app.world.get_resource::() == Some(&XrEnableStatus::Enabled) { + // app.world.get_resource::() == Some(&XrEnableStatus::Enabled) + if true { warn!("finished xr init"); let xr_instance = app .world @@ -152,7 +165,6 @@ impl Plugin for OpenXrPlugin { size: **xr_resolution, format: **xr_format, }; - app.add_systems(PreUpdate, xr_begin_frame.run_if(xr_only())); let mut manual_texture_views = app.world.resource_mut::(); manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); @@ -215,18 +227,13 @@ impl PluginGroup for DefaultXrPlugins { } } -pub fn xr_begin_frame( - instance: Res, - session: Res, +pub fn xr_poll_events( + instance: Option>, + session: Option>, session_running: Res, - mut frame_state: ResMut, - mut frame_waiter: ResMut, - swapchain: Res, - mut views: ResMut, - input: Res, mut app_exit: EventWriter, ) { - { + if let (Some(instance), Some(session)) = (instance, session) { let _span = info_span!("xr_poll_events"); while let Some(event) = instance.poll_event(&mut Default::default()).unwrap() { use xr::Event::*; @@ -237,6 +244,7 @@ pub fn xr_begin_frame( info!("entered XR state {:?}", e.state()); match e.state() { xr::SessionState::READY => { + info!("Calling Session begin :3"); session.begin(VIEW_TYPE).unwrap(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); } @@ -260,6 +268,16 @@ pub fn xr_begin_frame( } } } +} + +pub fn xr_begin_frame( + session: Res, + mut frame_state: ResMut, + mut frame_waiter: ResMut, + swapchain: Res, + mut views: ResMut, + input: Res, +) { { let _span = info_span!("xr_wait_frame").entered(); *frame_state = match frame_waiter.wait() { @@ -322,6 +340,8 @@ pub fn end_frame( swapchain: Option>, resolution: Option>, environment_blend_mode: Option>, + // _main_thread: NonSend<()>, + #[cfg(target_os = "android")] mut attached: Local, // passthrough_layer: Option>, ) { let xr_frame_state = xr_frame_state.unwrap(); @@ -330,6 +350,15 @@ pub fn end_frame( let swapchain = swapchain.unwrap(); let resolution = resolution.unwrap(); let environment_blend_mode = environment_blend_mode.unwrap(); + + #[cfg(target_os = "android")] + // if !*attached { + { + let ctx = ndk_context::android_context(); + let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); + let env = vm.attach_current_thread_as_daemon(); + *attached = true; + } { let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); diff --git a/src/resources.rs b/src/resources.rs index b73cb1c..584cb16 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -45,6 +45,7 @@ impl Plugin for XrResourcePlugin { app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); } } @@ -218,7 +219,8 @@ impl SwapchainInner { // } // None => - self.stream.lock().unwrap().end( + info!("swapchain stream lock"); + let r = self.stream.lock().unwrap().end( predicted_display_time, environment_blend_mode, &[&xr::CompositionLayerProjection::new().space(stage).views(&[ @@ -241,7 +243,9 @@ impl SwapchainInner { .image_rect(rect), ), ])], - ) + ); + info!("swapchain stream done"); + r // } } } diff --git a/src/xr_init.rs b/src/xr_init.rs index 13ee973..fb4917a 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -114,7 +114,14 @@ impl Plugin for RenderRestartPlugin { .add_event::() .add_event::() .add_event::() - .add_systems(PostStartup, setup_xr.run_if(xr_only())) + .add_systems( + PreUpdate, + setup_xr + .run_if(|running: Res| { + running.load(std::sync::atomic::Ordering::Relaxed) + }) + .run_if(run_once()), + ) .add_systems( PostUpdate, update_xr_stuff.run_if(on_event::()), @@ -132,8 +139,14 @@ impl Plugin for RenderRestartPlugin { .chain(), ) .add_systems(XrCleanup, cleanup_oxr_session); - app.add_systems(PostUpdate, start_xr_session.run_if(on_event::())); - app.add_systems(PostUpdate, stop_xr_session.run_if(on_event::())); + app.add_systems( + PostUpdate, + start_xr_session.run_if(on_event::()), + ); + app.add_systems( + PostUpdate, + stop_xr_session.run_if(on_event::()), + ); } } @@ -162,6 +175,8 @@ fn add_schedules(app: &mut App) { } pub fn setup_xr(world: &mut World) { + info!("running setup schedule :3"); + world.insert_resource(XrEnableStatus::Enabled); world.run_schedule(XrPreSetup); world.run_schedule(XrSetup); world.run_schedule(XrPrePostSetup); From cdeb7bda54dea78f9b4f03216212f9a99a792655 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 15 Jan 2024 05:13:38 +0100 Subject: [PATCH 14/80] fix android --- Cargo.toml | 4 ++++ src/lib.rs | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 2836170..77fb63a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,10 @@ openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features "static", ] } +[target.'cfg(target_os = "android")'.dependencies] +ndk-context = "0.1" +jni = "0.20" + [dev-dependencies] bevy = "0.12" color-eyre = "0.6.2" diff --git a/src/lib.rs b/src/lib.rs index 9a77e87..fed5e5b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -395,6 +395,13 @@ pub fn end_frame( environment_blend_mode: Res, // passthrough_layer: Option>, ) { + #[cfg(target_os = "android")] + { + let ctx = ndk_context::android_context(); + let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); + let env = vm.attach_current_thread_as_daemon(); + } + { let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); From 6f6948e6271010371594d1c2964e933a49fadfbe Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 23 Jan 2024 05:45:48 +0100 Subject: [PATCH 15/80] borked --- examples/demo/manifest.yaml | 1 + examples/demo/src/lib.rs | 28 +-- src/graphics/mod.rs | 2 +- src/lib.rs | 296 ++++++++++++------------ src/resources.rs | 6 +- src/xr_init.rs | 433 ------------------------------------ src/xr_input/hands/mod.rs | 46 +++- src/xr_input/mod.rs | 16 +- src/xr_input/xr_camera.rs | 66 ++++-- 9 files changed, 264 insertions(+), 630 deletions(-) delete mode 100644 src/xr_init.rs diff --git a/examples/demo/manifest.yaml b/examples/demo/manifest.yaml index 730187a..6cba5b8 100644 --- a/examples/demo/manifest.yaml +++ b/examples/demo/manifest.yaml @@ -13,6 +13,7 @@ android: # required: true uses_permission: - name: "com.oculus.permission.HAND_TRACKING" + - name: "android.permission.INTERNET" application: label: "Bevy Openxr Android" theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen" diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index 2867a23..e7feb94 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -19,7 +19,7 @@ use bevy_oxr::{ graphics::{extensions::XrExtensions, XrAppInfo, XrPreferdBlendMode}, input::XrInput, resources::{XrFrameState, XrInstance, XrSession}, - xr_init::{xr_only, XrEnableRequest, XrEnableStatus}, + xr_init::{xr_only, XrStatus}, xr_input::{ actions::XrActionSets, debug_gizmos::OpenXrDebugRenderer, @@ -41,18 +41,18 @@ use bevy_oxr::{ DefaultXrPlugins, }; -fn input_stuff( - keys: Res>, - status: Res, - mut request: EventWriter, -) { - if keys.just_pressed(KeyCode::Space) { - match status.into_inner() { - XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable), - XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable), - } - } -} +// fn input_stuff( +// keys: Res>, +// status: Res, +// mut request: EventWriter, +// ) { +// if keys.just_pressed(KeyCode::Space) { +// match status.into_inner() { +// XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable), +// XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable), +// } +// } +// } mod setup; use crate::setup::setup_scene; @@ -66,7 +66,7 @@ pub fn main() { let mut app = App::new(); let mut xr_extensions = XrExtensions::default(); - app.add_systems(Update, input_stuff) + app //lets get the usual diagnostic stuff added .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 24e80ff..33dc317 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -103,7 +103,7 @@ pub fn try_full_init( )> { let mut system_state: SystemState>> = SystemState::new(world); - let primary_window = system_state.get(&world).get_single().ok().cloned(); + let primary_window = system_state.get(world).get_single().ok().cloned(); let ( xr_instance, setup_info, diff --git a/src/lib.rs b/src/lib.rs index 4705e54..3b52be6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,10 +6,13 @@ pub mod resources; pub mod xr_init; pub mod xr_input; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::net::TcpStream; use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; -use crate::xr_init::RenderRestartPlugin; +use crate::xr_init::{StartXrSession, XrInitPlugin}; use crate::xr_input::hands::hand_tracking::DisableHandTracking; use crate::xr_input::oculus_touch::ActionSets; use bevy::app::{AppExit, PluginGroupBuilder}; @@ -18,22 +21,25 @@ use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; -use bevy::render::renderer::{render_system, RenderInstance}; +use bevy::render::renderer::RenderInstance; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; -use eyre::anyhow; use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; use input::XrInput; use openxr as xr; // use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; use resources::*; -use xr::{FormFactor, FrameWaiter}; -use xr_init::{xr_only, XrEnableStatus, XrRenderData}; +use xr::FormFactor; +use xr_init::{ + xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, +}; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin}; +use xr_input::hands::XrHandPlugins; +use xr_input::xr_camera::XrCameraType; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -49,141 +55,119 @@ pub struct OpenXrPlugin { app_info: XrAppInfo, } -#[derive(Resource)] -pub struct FutureXrResources( - pub Arc< - Mutex< - Option<( - XrInstance, - XrSession, - XrEnvironmentBlendMode, - XrResolution, - XrFormat, - XrSessionRunning, - XrSwapchain, - XrInput, - XrViews, - XrFrameState, - )>, - >, - >, -); - impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { app.insert_resource(XrSessionRunning::new(AtomicBool::new(false))); - // #[cfg(target_os = "android")] - // { - // let ctx = ndk_context::android_context(); - // let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); - // let env = vm.attach_current_thread_permanently(); - // } #[cfg(not(target_arch = "wasm32"))] - match graphics::try_full_init( - &mut app.world, + match graphics::initialize_xr_instance( + SystemState::>>::new(&mut app.world) + .get(&app.world) + .get_single() + .ok() + .cloned(), self.reqeusted_extensions.clone(), self.prefered_blend_mode, self.app_info.clone(), ) { - Ok((device, queue, adapter_info, render_adapter, instance)) => { + Ok(( + xr_instance, + oxr_session_setup_info, + blend_mode, + device, + queue, + adapter_info, + render_adapter, + instance, + )) => { debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); debug!("Configured wgpu adapter Features: {:#?}", device.features()); - warn!("Starting in Xr"); + warn!("Starting with OpenXR Instance"); app.insert_resource(ActionSets(vec![])); + app.insert_resource(xr_instance); + app.insert_resource(blend_mode); + app.insert_non_send_resource(oxr_session_setup_info); + let render_instance = RenderInstance(instance.into()); + app.insert_resource(render_instance.clone()); app.add_plugins(RenderPlugin { render_creation: RenderCreation::Manual( device, queue, adapter_info, render_adapter, - instance, + render_instance, ), }); - app.insert_resource(XrEnableStatus::Disabled); + app.insert_resource(XrStatus::Disabled); + app.world.send_event(StartXrSession); } Err(err) => { - warn!("OpenXR Failed to initialize: {}", err); + warn!("OpenXR Instance Failed to initialize: {}", err); app.add_plugins(RenderPlugin::default()); - app.add_plugins(ExtractResourcePlugin::::default()); - app.insert_resource(XrEnableStatus::Disabled); + app.insert_resource(XrStatus::NoInstance); } } - // app.add_systems(PreUpdate, mr_test); #[cfg(target_arch = "wasm32")] { app.add_plugins(RenderPlugin::default()); - app.insert_resource(XrEnableStatus::Disabled); + app.insert_resource(XrStatus::Disabled); } - app.add_plugins(ExtractResourcePlugin::::default()); app.add_systems( PreUpdate, - (xr_poll_events, xr_begin_frame.run_if(xr_only())).chain(), + xr_poll_events.run_if(|status: Res| *status != XrStatus::NoInstance), ); - } - - fn finish(&self, app: &mut App) { - // TODO: Split this up into the indevidual resources - // app.world.get_resource::() == Some(&XrEnableStatus::Enabled) - if true { - warn!("finished xr init"); - let xr_instance = app - .world - .get_resource::() - .expect("should exist"); - let xr_session = app.world.get_resource::().expect("should exist"); - let hands = xr_instance.exts().ext_hand_tracking.is_some() - && xr_instance - .supports_hand_tracking( - xr_instance - .system(FormFactor::HEAD_MOUNTED_DISPLAY) - .unwrap(), - ) - .is_ok_and(|v| v); - if hands { - app.insert_resource(HandTrackingData::new(xr_session).unwrap()); - } else { - app.insert_resource(DisableHandTracking::Both); - } - let xr_swapchain = app - .world - .get_resource::() - .expect("should exist"); - let xr_resolution = app - .world - .get_resource::() - .expect("should exist"); - let xr_format = app.world.get_resource::().expect("should exist"); - - let (left, right) = xr_swapchain.get_render_views(); - let left = ManualTextureView { - texture_view: left.into(), - size: **xr_resolution, - format: **xr_format, - }; - let right = ManualTextureView { - texture_view: right.into(), - size: **xr_resolution, - format: **xr_format, - }; - let mut manual_texture_views = app.world.resource_mut::(); - manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); - manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); - drop(manual_texture_views); - let render_app = app.sub_app_mut(RenderApp); - render_app.add_systems( - Render, - ( - post_frame - .run_if(xr_only()) - .before(render_system) - .after(RenderSet::ExtractCommands), - end_frame.run_if(xr_only()).after(render_system), - ), - ); - } + app.add_systems( + PreUpdate, + ( + xr_reset_should_render, + apply_deferred, + xr_wait_frame.run_if(xr_only()), + apply_deferred, + locate_views.run_if(xr_only()), + apply_deferred, + ) + .chain() + .after(xr_poll_events), + ); + let render_app = app.sub_app_mut(RenderApp); + render_app.add_systems( + Render, + xr_begin_frame + .run_if(xr_only()) + .run_if(xr_render_only()) + .after(RenderSet::ExtractCommands) + .before(xr_pre_frame), + ); + render_app.add_systems( + Render, + xr_pre_frame + .run_if(xr_only()) + .run_if(xr_render_only()) + .in_set(RenderSet::Prepare), + ); + render_app.add_systems( + Render, + (locate_views, xr_input::xr_camera::xr_camera_head_sync) + .chain() + .run_if(xr_only()) + .run_if(xr_render_only()) + .in_set(RenderSet::Prepare), + ); + render_app.add_systems( + Render, + xr_end_frame + .run_if(xr_only()) + .run_if(xr_render_only()) + .after(RenderSet::Render), + ); + render_app.insert_resource(TcpConnection( + TcpStream::connect("192.168.2.100:6969").unwrap(), + )); } } +#[derive(Resource)] +struct TcpConnection(TcpStream); + #[derive(Default)] pub struct DefaultXrPlugins { pub reqeusted_extensions: XrExtensions, @@ -203,11 +187,10 @@ impl PluginGroup for DefaultXrPlugins { app_info: self.app_info.clone(), }) .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) - .add_before::(RenderRestartPlugin) - .add(HandEmulationPlugin) - .add(HandTrackingPlugin) + .add_after::(XrInitPlugin) + .add_before::(XrEarlyInitPlugin) + .add(XrHandPlugins) .add(XrResourcePlugin) - // .add(xr_init::RenderRestartPlugin) .set(WindowPlugin { #[cfg(not(target_os = "android"))] primary_window: Some(Window { @@ -227,11 +210,16 @@ impl PluginGroup for DefaultXrPlugins { } } -pub fn xr_poll_events( +fn xr_reset_should_render(mut should: ResMut) { + **should = false; +} + +fn xr_poll_events( instance: Option>, session: Option>, session_running: Res, mut app_exit: EventWriter, + mut cleanup_xr: EventWriter, ) { if let (Some(instance), Some(session)) = (instance, session) { let _span = info_span!("xr_poll_events"); @@ -251,6 +239,7 @@ pub fn xr_poll_events( xr::SessionState::STOPPING => { session.end().unwrap(); session_running.store(false, std::sync::atomic::Ordering::Relaxed); + cleanup_xr.send_default(); } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { // app_exit.send(AppExit); @@ -260,7 +249,9 @@ pub fn xr_poll_events( _ => {} } } - InstanceLossPending(_) => return, + InstanceLossPending(_) => { + app_exit.send_default(); + } EventsLost(e) => { warn!("lost {} XR events", e.lost_event_count()); } @@ -270,13 +261,15 @@ pub fn xr_poll_events( } } -pub fn xr_begin_frame( - session: Res, +fn xr_begin_frame(swapchain: Res) { + let _span = info_span!("xr_begin_frame").entered(); + swapchain.begin().unwrap() +} + +pub fn xr_wait_frame( mut frame_state: ResMut, mut frame_waiter: ResMut, - swapchain: Res, - mut views: ResMut, - input: Res, + mut should_render: ResMut, ) { { let _span = info_span!("xr_wait_frame").entered(); @@ -287,21 +280,11 @@ pub fn xr_begin_frame( return; } }; - } - { - let _span = info_span!("xr_begin_frame").entered(); - swapchain.begin().unwrap() - } - { - let _span = info_span!("xr_locate_views").entered(); - **views = session - .locate_views(VIEW_TYPE, frame_state.predicted_display_time, &input.stage) - .unwrap() - .1; + **should_render = frame_state.should_render; } } -pub fn post_frame( +pub fn xr_pre_frame( resolution: Res, format: Res, swapchain: Res, @@ -313,7 +296,9 @@ pub fn post_frame( } { let _span = info_span!("xr_wait_image").entered(); + info!("wait image"); swapchain.wait_image().unwrap(); + info!("waited image"); } { let _span = info_span!("xr_update_manual_texture_views").entered(); @@ -333,36 +318,51 @@ pub fn post_frame( } } -pub fn end_frame( - xr_frame_state: Option>, - views: Option>, - input: Option>, - swapchain: Option>, - resolution: Option>, - environment_blend_mode: Option>, - // _main_thread: NonSend<()>, - #[cfg(target_os = "android")] mut attached: Local, - // passthrough_layer: Option>, +pub fn xr_end_frame( + xr_frame_state: Res, + views: Res, + input: Res, + swapchain: Res, + resolution: Res, + environment_blend_mode: Res, + mut connection: ResMut, + cams: Query<(&Transform, &XrCameraType)>, ) { - let xr_frame_state = xr_frame_state.unwrap(); - let views = views.unwrap(); - let input = input.unwrap(); - let swapchain = swapchain.unwrap(); - let resolution = resolution.unwrap(); - let environment_blend_mode = environment_blend_mode.unwrap(); - #[cfg(target_os = "android")] - // if !*attached { { let ctx = ndk_context::android_context(); let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); let env = vm.attach_current_thread_as_daemon(); - *attached = true; } { let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); } + let mut cam = None; + for (t, c) in &cams { + if *c == XrCameraType::Xr(xr_input::xr_camera::Eye::Left) { + cam = Some(*t); + break; + } + } + let _ = std::writeln!( + &mut connection.0, + "{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + views[0].pose.position.x, + views[0].pose.position.y, + views[0].pose.position.z, + views[0].pose.orientation.x, + views[0].pose.orientation.y, + views[0].pose.orientation.z, + views[0].pose.orientation.w, + cam.unwrap().translation.x, + cam.unwrap().translation.y, + cam.unwrap().translation.z, + cam.unwrap().rotation.x, + cam.unwrap().rotation.y, + cam.unwrap().rotation.z, + cam.unwrap().rotation.w, + ); { let _span = info_span!("xr_end_frame").entered(); let result = swapchain.end( diff --git a/src/resources.rs b/src/resources.rs index 584cb16..da72cf6 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -5,7 +5,7 @@ use std::sync::Mutex; use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; use crate::resource_macros::*; -use crate::xr_init::XrEnableStatus; +use crate::xr_init::XrStatus; use bevy::prelude::*; use bevy::render::extract_resource::ExtractResourcePlugin; use openxr as xr; @@ -27,7 +27,6 @@ pub(crate) struct VulkanOXrSessionSetupInfo { pub(crate) vk_instance_ptr: *const c_void, pub(crate) queue_family_index: u32, pub(crate) xr_system_id: xr::SystemId, - // pub(crate) swapchain_create_info: xr::SwapchainCreateInfo } pub(crate) enum OXrSessionSetupInfo { @@ -46,6 +45,7 @@ impl Plugin for XrResourcePlugin { app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); } } @@ -219,7 +219,6 @@ impl SwapchainInner { // } // None => - info!("swapchain stream lock"); let r = self.stream.lock().unwrap().end( predicted_display_time, environment_blend_mode, @@ -244,7 +243,6 @@ impl SwapchainInner { ), ])], ); - info!("swapchain stream done"); r // } } diff --git a/src/xr_init.rs b/src/xr_init.rs deleted file mode 100644 index fb4917a..0000000 --- a/src/xr_init.rs +++ /dev/null @@ -1,433 +0,0 @@ -// Just a lot of code that is meant for something way more complex but hey. -// maybe will work on that soon - -use std::sync::Arc; - -use bevy::{ - ecs::schedule::{ExecutorKind, ScheduleLabel}, - prelude::*, - render::{ - extract_resource::{ExtractResource, ExtractResourcePlugin}, - renderer::{ - self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, - }, - settings::WgpuSettings, - }, - window::{PrimaryWindow, RawHandleWrapper}, -}; -use wgpu::Instance; - -use crate::{ - graphics, - input::XrInput, - resources::{ - OXrSessionSetupInfo, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, - XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, - }, -}; - -#[derive(Resource, Clone)] -pub struct RenderCreationData { - pub device: RenderDevice, - pub queue: RenderQueue, - pub adapter_info: RenderAdapterInfo, - pub render_adapter: RenderAdapter, - pub instance: Arc, -} - -#[derive(Resource, Clone, ExtractResource)] -pub struct XrRenderData { - pub xr_instance: XrInstance, - pub xr_session: XrSession, - pub xr_blend_mode: XrEnvironmentBlendMode, - pub xr_resolution: XrResolution, - pub xr_format: XrFormat, - pub xr_session_running: XrSessionRunning, - // pub xr_frame_waiter: XrFrameWaiter, - pub xr_swapchain: XrSwapchain, - pub xr_input: XrInput, - pub xr_views: XrViews, - pub xr_frame_state: XrFrameState, -} - -#[derive(Event, Clone, Copy, Debug)] -pub enum XrEnableRequest { - TryEnable, - TryDisable, -} -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Reflect, ExtractResource)] -pub enum XrEnableStatus { - Enabled, - Disabled, -} - -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Debug, ExtractResource)] -pub enum XrNextEnabledState { - Enabled, - Disabled, -} - -pub struct RenderRestartPlugin; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPreSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPrePostSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPostSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPreCleanup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrCleanup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPostCleanup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPreRenderUpdate; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrRenderUpdate; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPostRenderUpdate; - -pub fn xr_only() -> impl FnMut(Option>, Res) -> bool { - |status, running| { - resource_exists_and_equals(XrEnableStatus::Enabled)(status) - && running.load(std::sync::atomic::Ordering::Relaxed) - } -} - -impl Plugin for RenderRestartPlugin { - fn build(&self, app: &mut App) { - add_schedules(app); - app.add_plugins(ExtractResourcePlugin::::default()) - .add_event::() - .add_event::() - .add_event::() - .add_event::() - .add_systems( - PreUpdate, - setup_xr - .run_if(|running: Res| { - running.load(std::sync::atomic::Ordering::Relaxed) - }) - .run_if(run_once()), - ) - .add_systems( - PostUpdate, - update_xr_stuff.run_if(on_event::()), - ) - .add_systems(XrPreRenderUpdate, decide_next_xr_state) - .add_systems(XrPostRenderUpdate, clear_events) - .add_systems( - XrRenderUpdate, - ( - cleanup_xr.run_if(resource_exists_and_equals(XrNextEnabledState::Disabled)), - // handle_xr_enable_requests, - apply_deferred, - setup_xr, /* .run_if(resource_exists_and_equals(XrEnableStatus::Enabled)) */ - ) - .chain(), - ) - .add_systems(XrCleanup, cleanup_oxr_session); - app.add_systems( - PostUpdate, - start_xr_session.run_if(on_event::()), - ); - app.add_systems( - PostUpdate, - stop_xr_session.run_if(on_event::()), - ); - } -} - -fn clear_events(mut events: ResMut>) { - events.clear(); -} - -fn add_schedules(app: &mut App) { - let schedules = [ - Schedule::new(XrPreSetup), - Schedule::new(XrSetup), - Schedule::new(XrPrePostSetup), - Schedule::new(XrPostSetup), - Schedule::new(XrPreRenderUpdate), - Schedule::new(XrRenderUpdate), - Schedule::new(XrPostRenderUpdate), - Schedule::new(XrPreCleanup), - Schedule::new(XrCleanup), - Schedule::new(XrPostCleanup), - ]; - for mut schedule in schedules { - schedule.set_executor_kind(ExecutorKind::SingleThreaded); - schedule.set_apply_final_deferred(true); - app.add_schedule(schedule); - } -} - -pub fn setup_xr(world: &mut World) { - info!("running setup schedule :3"); - world.insert_resource(XrEnableStatus::Enabled); - world.run_schedule(XrPreSetup); - world.run_schedule(XrSetup); - world.run_schedule(XrPrePostSetup); - world.run_schedule(XrPostSetup); -} -fn cleanup_xr(world: &mut World) { - world.run_schedule(XrPreCleanup); - world.run_schedule(XrCleanup); - world.run_schedule(XrPostCleanup); -} - -fn cleanup_oxr_session(xr_status: Option>, session: Option>) { - if let (Some(XrEnableStatus::Disabled), Some(s)) = (xr_status.map(|v| v.into_inner()), session) - { - s.into_inner().request_exit().unwrap(); - } -} - -pub fn update_xr_stuff(world: &mut World) { - world.run_schedule(XrPreRenderUpdate); - world.run_schedule(XrRenderUpdate); - world.run_schedule(XrPostRenderUpdate); -} - -#[derive(Event, Clone, Copy, Default)] -pub struct StartXrSession; - -#[derive(Event, Clone, Copy, Default)] -pub struct EndXrSession; - -fn start_xr_session( - mut commands: Commands, - instance: Res, - primary_window: Query<&RawHandleWrapper, With>, - setup_info: NonSend, - render_device: Res, - render_adapter: Res, - render_instance: Res, -) { - let ( - xr_session, - xr_resolution, - xr_format, - xr_session_running, - xr_frame_waiter, - xr_swapchain, - xr_input, - xr_views, - xr_frame_state, - ) = match graphics::start_xr_session( - primary_window.get_single().cloned().ok(), - &setup_info, - &instance, - &render_device, - &render_adapter, - &render_instance, - ) { - Ok(data) => data, - Err(err) => { - error!("Unable to start OpenXR Session: {}", err); - return; - } - }; - commands.insert_resource(xr_session); - commands.insert_resource(xr_resolution); - commands.insert_resource(xr_format); - commands.insert_resource(xr_session_running); - commands.insert_resource(xr_frame_waiter); - commands.insert_resource(xr_swapchain); - commands.insert_resource(xr_input); - commands.insert_resource(xr_views); - commands.insert_resource(xr_frame_state); -} - -fn stop_xr_session(mut commands: Commands, session: ResMut) { - match session.request_exit() { - Ok(_) => {} - Err(err) => { - error!("Error while trying to request session exit: {}", err) - } - } -} - -// fn handle_xr_enable_requests( -// primary_window: Query<&RawHandleWrapper, With>, -// mut commands: Commands, -// next_state: Res, -// on_main: Option>, -// ) { -// // Just to force this system onto the main thread because of unsafe code -// let _ = on_main; -// -// let (creation_data, xr_data) = match next_state.into_inner() { -// XrNextEnabledState::Enabled => { -// let ( -// device, -// queue, -// adapter_info, -// render_adapter, -// instance, -// xr_instance, -// session, -// blend_mode, -// resolution, -// format, -// session_running, -// frame_waiter, -// swapchain, -// input, -// views, -// frame_state, -// ) = graphics::initialize_xr_graphics(primary_window.get_single().ok().cloned()) -// .unwrap(); -// -// commands.insert_resource(XrEnableStatus::Enabled); -// ( -// RenderCreationData { -// device, -// queue, -// adapter_info, -// render_adapter, -// instance: Arc::new(instance), -// }, -// Some(XrRenderData { -// xr_instance, -// xr_session: session, -// xr_blend_mode: blend_mode, -// xr_resolution: resolution, -// xr_format: format, -// xr_session_running: session_running, -// xr_frame_waiter: frame_waiter, -// xr_swapchain: swapchain, -// xr_input: input, -// xr_views: views, -// xr_frame_state: frame_state, -// }), -// ) -// } -// XrNextEnabledState::Disabled => ( -// init_non_xr_graphics(primary_window.get_single().ok().cloned()), -// None, -// ), -// }; -// -// commands.insert_resource(creation_data.device); -// commands.insert_resource(creation_data.queue); -// commands.insert_resource(RenderInstance(creation_data.instance)); -// commands.insert_resource(creation_data.adapter_info); -// commands.insert_resource(creation_data.render_adapter); -// -// if let Some(xr_data) = xr_data { -// // TODO: Remove this lib.rs:144 -// commands.insert_resource(xr_data.clone()); -// -// commands.insert_resource(xr_data.xr_instance); -// commands.insert_resource(xr_data.xr_session); -// commands.insert_resource(xr_data.xr_blend_mode); -// commands.insert_resource(xr_data.xr_resolution); -// commands.insert_resource(xr_data.xr_format); -// commands.insert_resource(xr_data.xr_session_running); -// commands.insert_resource(xr_data.xr_frame_waiter); -// commands.insert_resource(xr_data.xr_input); -// commands.insert_resource(xr_data.xr_views); -// commands.insert_resource(xr_data.xr_frame_state); -// commands.insert_resource(xr_data.xr_swapchain); -// } else { -// commands.remove_resource::(); -// -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// } -// } - -fn decide_next_xr_state( - mut commands: Commands, - mut events: EventReader, - xr_status: Option>, -) { - info!("hm"); - let request = match events.read().next() { - Some(v) => v, - None => return, - }; - info!("ok"); - match (request, xr_status.as_deref()) { - (XrEnableRequest::TryEnable, Some(XrEnableStatus::Enabled)) => { - info!("Xr Already Enabled! ignoring request"); - return; - } - (XrEnableRequest::TryDisable, Some(XrEnableStatus::Disabled)) => { - info!("Xr Already Disabled! ignoring request"); - return; - } - // (_, Some(XrEnableStatus::Waiting)) => { - // info!("Already Handling Request! ignoring request"); - // return; - // } - _ => {} - } - let r = match request { - XrEnableRequest::TryEnable => XrNextEnabledState::Enabled, - XrEnableRequest::TryDisable => XrNextEnabledState::Disabled, - }; - info!("{:#?}", r); - commands.insert_resource(r); -} - -pub fn init_non_xr_graphics(primary_window: Option) -> RenderCreationData { - let settings = WgpuSettings::default(); - - let async_renderer = async move { - let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { - // Probably a bad idea unwraping here but on the other hand no backends - backends: settings.backends.unwrap(), - dx12_shader_compiler: settings.dx12_shader_compiler.clone(), - }); - let surface = primary_window.map(|wrapper| unsafe { - // SAFETY: Plugins should be set up on the main thread. - let handle = wrapper.get_handle(); - instance - .create_surface(&handle) - .expect("Failed to create wgpu surface") - }); - - let request_adapter_options = wgpu::RequestAdapterOptions { - power_preference: settings.power_preference, - compatible_surface: surface.as_ref(), - ..Default::default() - }; - - let (device, queue, adapter_info, render_adapter) = - renderer::initialize_renderer(&instance, &settings, &request_adapter_options).await; - debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); - debug!("Configured wgpu adapter Features: {:#?}", device.features()); - RenderCreationData { - device, - queue, - adapter_info, - render_adapter, - instance: Arc::new(instance), - } - }; - // No need for wasm in bevy_oxr web xr would be a different crate - futures_lite::future::block_on(async_renderer) -} diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs index 06eb5c1..fa85cf8 100644 --- a/src/xr_input/hands/mod.rs +++ b/src/xr_input/hands/mod.rs @@ -1,6 +1,15 @@ use bevy::{app::PluginGroupBuilder, prelude::*}; +use openxr::FormFactor; -use self::{emulated::HandEmulationPlugin, hand_tracking::HandTrackingPlugin}; +use crate::{ + resources::{XrInstance, XrSession}, + xr_init::XrPreSetup, +}; + +use self::{ + emulated::HandEmulationPlugin, + hand_tracking::{DisableHandTracking, HandTrackingData, HandTrackingPlugin}, +}; pub mod common; pub mod emulated; @@ -8,12 +17,35 @@ pub mod hand_tracking; pub struct XrHandPlugins; -impl PluginGroup for XrHandPlugins { - fn build(self) -> PluginGroupBuilder { - PluginGroupBuilder::start::() - .add(HandTrackingPlugin) - .add(HandEmulationPlugin) - .build() +impl Plugin for XrHandPlugins { + fn build(&self, app: &mut App) { + app.add_plugins(HandTrackingPlugin) + .add_plugins(HandPlugin) + .add_plugins(HandEmulationPlugin); + } +} + +pub struct HandPlugin; + +impl Plugin for HandPlugin { + fn build(&self, app: &mut App) { + app.add_systems(XrPreSetup, check_for_handtracking); + } +} + +fn check_for_handtracking( + mut commands: Commands, + instance: Res, + session: Res, +) { + let hands = instance.exts().ext_hand_tracking.is_some() + && instance + .supports_hand_tracking(instance.system(FormFactor::HEAD_MOUNTED_DISPLAY).unwrap()) + .is_ok_and(|v| v); + if hands { + commands.insert_resource(HandTrackingData::new(&session).unwrap()); + } else { + commands.insert_resource(DisableHandTracking::Both); } } diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index 540f2e2..ac2e13b 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -10,11 +10,11 @@ pub mod trackers; pub mod xr_camera; use crate::resources::{XrInstance, XrSession}; -use crate::xr_begin_frame; -use crate::xr_init::{xr_only, XrPostSetup, XrSetup, XrPreSetup}; +use crate::xr_init::{xr_only, XrPostSetup, XrPreSetup, XrSetup}; use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; +use crate::{locate_views, xr_wait_frame}; use bevy::app::{App, PostUpdate, Startup}; use bevy::ecs::entity::Entity; use bevy::ecs::query::With; @@ -24,17 +24,19 @@ use bevy::math::Vec2; use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource}; use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3}; use bevy::render::camera::CameraProjectionPlugin; +use bevy::render::extract_component::ExtractComponentPlugin; use bevy::render::view::{update_frusta, VisibilitySystems}; use bevy::transform::TransformSystem; use bevy::utils::HashMap; use openxr::Binding; use self::actions::{setup_oxr_actions, OpenXrActionsPlugin}; -use self::oculus_touch::{post_action_setup_oculus_controller, ActionSets, init_subaction_path}; +use self::oculus_touch::{init_subaction_path, post_action_setup_oculus_controller, ActionSets}; use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, }; +use self::xr_camera::{XrCameraType, TransformExtract}; #[derive(Copy, Clone)] pub struct OpenXrInput { @@ -67,7 +69,10 @@ impl Plugin for OpenXrInput { app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); app.add_systems( PreUpdate, - xr_camera_head_sync.run_if(xr_only()).after(xr_begin_frame), + xr_camera_head_sync + .run_if(xr_only()) + .after(xr_wait_frame) + .after(locate_views), ); //update controller trackers app.add_systems(Update, update_open_xr_controllers.run_if(xr_only())); @@ -79,6 +84,9 @@ impl Plugin for OpenXrInput { ); app.add_systems(XrPreSetup, init_subaction_path); app.add_systems(XrSetup, setup_xr_cameras); + app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); } } diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 210feaa..1d143c3 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -1,9 +1,11 @@ use crate::xr_input::{QuatConv, Vec3Conv}; use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; +use bevy::ecs::system::lifetimeless::Read; use bevy::math::Vec3A; use bevy::prelude::*; use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; +use bevy::render::extract_component::ExtractComponent; use bevy::render::primitives::Frustum; use bevy::render::view::{ColorGrading, VisibleEntities}; use openxr::Fovf; @@ -48,6 +50,35 @@ pub enum XrCameraType { Flatscreen, } + +#[derive(Component)] +pub(super) struct TransformExtract; + + +impl ExtractComponent for TransformExtract { + type Query = Read; + + type Filter = (); + + type Out = Transform; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + Some(*item) + } +} + +impl ExtractComponent for XrCameraType { + type Query = Read; + + type Filter = (); + + type Out = Self; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + Some(*item) + } +} + #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, @@ -81,7 +112,7 @@ impl XrCameraBundle { } } -#[derive(Debug, Clone, Component, Reflect)] +#[derive(Debug, Clone, Component, Reflect, ExtractComponent)] #[reflect(Component, Default)] pub struct XRProjection { pub near: f32, @@ -241,24 +272,21 @@ impl CameraProjection for XRProjection { pub fn xr_camera_head_sync( views: ResMut, - query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, ) { - fn f( - views: ResMut, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, - ) -> Option<()> { - //TODO calculate HMD position - for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - XrCameraType::Xr(eye) => *eye as usize, - XrCameraType::Flatscreen => return None, - }; - let view = views.get(view_idx)?; - xr_projection.fov = view.fov; - transform.rotation = view.pose.orientation.to_quat(); - transform.translation = view.pose.position.to_vec3(); - } - Some(()) + //TODO calculate HMD position + for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { + let view_idx = match camera_type { + XrCameraType::Xr(eye) => *eye as usize, + // I don't belive we need a flatscrenn cam, that's just a cam without this component + XrCameraType::Flatscreen => continue, + }; + let view = match views.get(view_idx) { + Some(views) => views, + None => continue, + }; + xr_projection.fov = view.fov; + transform.rotation = view.pose.orientation.to_quat(); + transform.translation = view.pose.position.to_vec3(); } - let _ = f(views, query); } From a0ac82b21c3e79b46bb27eded4a560f888bf8e9d Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 23 Jan 2024 05:46:01 +0100 Subject: [PATCH 16/80] borked --- + | 294 +++++++++++++++++++++++++++++++++++++++ src/xr_init/mod.rs | 197 ++++++++++++++++++++++++++ src/xr_init/schedules.rs | 51 +++++++ 3 files changed, 542 insertions(+) create mode 100644 + create mode 100644 src/xr_init/mod.rs create mode 100644 src/xr_init/schedules.rs diff --git a/+ b/+ new file mode 100644 index 0000000..8e33f19 --- /dev/null +++ b/+ @@ -0,0 +1,294 @@ +use crate::xr_input::{QuatConv, Vec3Conv}; +use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; +use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; +use bevy::ecs::system::lifetimeless::Read; +use bevy::math::Vec3A; +use bevy::prelude::*; +use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; +use bevy::render::extract_component::ExtractComponent; +use bevy::render::primitives::Frustum; +use bevy::render::view::{ColorGrading, VisibleEntities}; +use openxr::Fovf; + +#[derive(Bundle)] +pub struct XrCamerasBundle { + pub left: XrCameraBundle, + pub right: XrCameraBundle, +} +impl XrCamerasBundle { + pub fn new() -> Self { + Self::default() + } +} +impl Default for XrCamerasBundle { + fn default() -> Self { + Self { + left: XrCameraBundle::new(Eye::Left), + right: XrCameraBundle::new(Eye::Right), + } + } +} + +#[derive(Bundle)] +pub struct XrCameraBundle { + pub camera: Camera, + pub camera_render_graph: CameraRenderGraph, + pub xr_projection: XRProjection, + pub visible_entities: VisibleEntities, + pub frustum: Frustum, + pub transform: Transform, + pub global_transform: GlobalTransform, + pub camera_3d: Camera3d, + pub tonemapping: Tonemapping, + pub dither: DebandDither, + pub color_grading: ColorGrading, + pub xr_camera_type: XrCameraType, +} +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] +pub enum XrCameraType { + Xr(Eye), + Flatscreen, +} + + +#[derive(Component)] +pub(super) struct TransformExtract; + + +impl ExtractComponent for TransformExtract { + type Query = Read; + + type Filter = (); + + type Out = Transform; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + info!("extracting Transform"); + Some(*item) + } +} + +impl ExtractComponent for XrCameraType { + type Query = Read; + + type Filter = (); + + type Out = Self; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + info!("extracting Cam Type"); + Some(*item) + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub enum Eye { + Left = 0, + Right = 1, +} + +impl XrCameraBundle { + pub fn new(eye: Eye) -> Self { + Self { + camera: Camera { + order: -1, + target: RenderTarget::TextureView(match eye { + Eye::Left => LEFT_XR_TEXTURE_HANDLE, + Eye::Right => RIGHT_XR_TEXTURE_HANDLE, + }), + viewport: None, + ..default() + }, + camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME), + xr_projection: Default::default(), + visible_entities: Default::default(), + frustum: Default::default(), + transform: Default::default(), + global_transform: Default::default(), + camera_3d: Default::default(), + tonemapping: Default::default(), + dither: DebandDither::Enabled, + color_grading: Default::default(), + xr_camera_type: XrCameraType::Xr(eye), + } + } +} + +#[derive(Debug, Clone, Component, Reflect, ExtractComponent)] +#[reflect(Component, Default)] +pub struct XRProjection { + pub near: f32, + pub far: f32, + #[reflect(ignore)] + pub fov: Fovf, +} + +impl Default for XRProjection { + fn default() -> Self { + Self { + near: 0.1, + far: 1000., + fov: Default::default(), + } + } +} + +impl XRProjection { + pub fn new(near: f32, far: f32, fov: Fovf) -> Self { + XRProjection { near, far, fov } + } +} + +impl CameraProjection for XRProjection { + // ============================================================================= + // math code adapted from + // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h + // Copyright (c) 2017 The Khronos Group Inc. + // Copyright (c) 2016 Oculus VR, LLC. + // SPDX-License-Identifier: Apache-2.0 + // ============================================================================= + fn get_projection_matrix(&self) -> Mat4 { + // symmetric perspective for debugging + // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); + // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); + // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); + + let fov = self.fov; + let is_vulkan_api = false; // FIXME wgpu probably abstracts this + let near_z = self.near; + let far_z = -1.; // use infinite proj + // let far_z = self.far; + + let tan_angle_left = fov.angle_left.tan(); + let tan_angle_right = fov.angle_right.tan(); + + let tan_angle_down = fov.angle_down.tan(); + let tan_angle_up = fov.angle_up.tan(); + + let tan_angle_width = tan_angle_right - tan_angle_left; + + // Set to tanAngleDown - tanAngleUp for a clip space with positive Y + // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with + // positive Y up (OpenGL / D3D / Metal). + // const float tanAngleHeight = + // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); + let tan_angle_height = if is_vulkan_api { + tan_angle_down - tan_angle_up + } else { + tan_angle_up - tan_angle_down + }; + + // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). + // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). + // const float offsetZ = + // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; + // FIXME handle enum of graphics apis + let offset_z = 0.; + + let mut cols: [f32; 16] = [0.0; 16]; + + if far_z <= near_z { + // place the far plane at infinity + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -1.; + cols[14] = -(near_z + offset_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + + // bevy uses the _reverse_ infinite projection + // https://dev.theomader.com/depth-precision/ + let z_reversal = Mat4::from_cols_array_2d(&[ + [1f32, 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., -1., 0.], + [0., 0., 1., 1.], + ]); + + return z_reversal * Mat4::from_cols_array(&cols); + } else { + // normal projection + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -(far_z + offset_z) / (far_z - near_z); + cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + } + + Mat4::from_cols_array(&cols) + } + + fn update(&mut self, _width: f32, _height: f32) {} + + fn far(&self) -> f32 { + self.far + } + + fn get_frustum_corners(&self, z_near: f32, z_far: f32) -> [Vec3A; 8] { + let tan_angle_left = self.fov.angle_left.tan(); + let tan_angle_right = self.fov.angle_right.tan(); + + let tan_angle_bottom = self.fov.angle_down.tan(); + let tan_angle_top = self.fov.angle_up.tan(); + + // NOTE: These vertices are in the specific order required by [`calculate_cascade`]. + [ + Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_near, // bottom right + Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_near, // top right + Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_near, // top left + Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_near, // bottom left + Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_far, // bottom right + Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_far, // top right + Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_far, // top left + Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_far, // bottom left + ] + } +} + +pub fn xr_camera_head_sync( + views: ResMut, + mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, +) { + //TODO calculate HMD position + for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { + let view_idx = match camera_type { + XrCameraType::Xr(eye) => *eye as usize, + // I don't belive we need a flatscrenn cam, that's just a cam without this component + XrCameraType::Flatscreen => continue, + }; + let view = match views.get(view_idx) { + Some(views) => views, + None => continue, + }; + xr_projection.fov = view.fov; + transform.rotation = view.pose.orientation.to_quat(); + transform.translation = view.pose.position.to_vec3(); + } +} diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs new file mode 100644 index 0000000..d7db1c9 --- /dev/null +++ b/src/xr_init/mod.rs @@ -0,0 +1,197 @@ +pub mod schedules; +pub use schedules::*; + +use bevy::{ + prelude::*, + render::{ + camera::{ManualTextureView, ManualTextureViews}, + extract_resource::{ExtractResource, ExtractResourcePlugin}, + renderer::{RenderAdapter, RenderDevice, RenderInstance}, + }, + window::{PrimaryWindow, RawHandleWrapper}, +}; + +use crate::{ + graphics, + resources::{ + OXrSessionSetupInfo, XrFormat, XrInstance, XrResolution, XrSession, XrSessionRunning, + XrSwapchain, + }, + LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE, +}; + +#[derive(Resource, Event, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource)] +pub enum XrStatus { + NoInstance, + Enabled, + Enabling, + Disabled, + Disabling, +} + +#[derive( + Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut, +)] +pub struct XrShouldRender(bool); + +pub struct XrEarlyInitPlugin; + +pub struct XrInitPlugin; + +pub fn xr_only() -> impl FnMut(Option>) -> bool { + resource_exists_and_equals(XrStatus::Enabled) +} +pub fn xr_render_only() -> impl FnMut(Option>) -> bool { + resource_exists_and_equals(XrShouldRender(true)) +} + +impl Plugin for XrEarlyInitPlugin { + fn build(&self, app: &mut App) { + app.add_event::() + .add_event::() + .add_event::() + .add_event::(); + } +} + +impl Plugin for XrInitPlugin { + fn build(&self, app: &mut App) { + add_schedules(app); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.init_resource::(); + app.add_systems(PreUpdate, setup_xr.run_if(on_event::())) + .add_systems(PreUpdate, cleanup_xr.run_if(on_event::())); + app.add_systems( + PostUpdate, + start_xr_session.run_if(on_event::()), + ); + app.add_systems( + PostUpdate, + stop_xr_session.run_if(on_event::()), + ); + app.add_systems(XrSetup, setup_manual_texture_views); + } +} + +fn setup_manual_texture_views( + mut manual_texture_views: ResMut, + swapchain: Res, + xr_resolution: Res, + xr_format: Res, +) { + info!("Creating Texture views"); + let (left, right) = swapchain.get_render_views(); + let left = ManualTextureView { + texture_view: left.into(), + size: **xr_resolution, + format: **xr_format, + }; + let right = ManualTextureView { + texture_view: right.into(), + size: **xr_resolution, + format: **xr_format, + }; + manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); + manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); +} + +pub fn setup_xr(world: &mut World) { + world.run_schedule(XrPreSetup); + world.run_schedule(XrSetup); + world.run_schedule(XrPrePostSetup); + world.run_schedule(XrPostSetup); + *world.resource_mut::() = XrStatus::Enabled; +} +fn cleanup_xr(world: &mut World) { + world.run_schedule(XrPreCleanup); + world.run_schedule(XrCleanup); + world.run_schedule(XrPostCleanup); + *world.resource_mut::() = XrStatus::Disabled; +} + +#[derive(Event, Clone, Copy, Default)] +pub struct StartXrSession; + +#[derive(Event, Clone, Copy, Default)] +pub struct EndXrSession; + +#[derive(Event, Clone, Copy, Default)] +struct SetupXrData; +#[derive(Event, Clone, Copy, Default)] +pub(crate) struct CleanupXrData; + +#[allow(clippy::too_many_arguments)] +fn start_xr_session( + mut commands: Commands, + mut setup_xr: EventWriter, + mut status: ResMut, + instance: Res, + primary_window: Query<&RawHandleWrapper, With>, + setup_info: NonSend, + render_device: Res, + render_adapter: Res, + render_instance: Res, +) { + info!("start Session"); + match *status { + XrStatus::Disabled => {} + XrStatus::NoInstance => { + warn!("Trying to start OpenXR Session without instance, ignoring"); + return; + } + XrStatus::Enabled | XrStatus::Enabling => { + warn!("Trying to start OpenXR Session while one already exists, ignoring"); + return; + } + XrStatus::Disabling => { + warn!("Trying to start OpenXR Session while one is stopping, ignoring"); + return; + } + } + let ( + xr_session, + xr_resolution, + xr_format, + xr_session_running, + xr_frame_waiter, + xr_swapchain, + xr_input, + xr_views, + xr_frame_state, + ) = match graphics::start_xr_session( + primary_window.get_single().cloned().ok(), + &setup_info, + &instance, + &render_device, + &render_adapter, + &render_instance, + ) { + Ok(data) => data, + Err(err) => { + error!("Unable to start OpenXR Session: {}", err); + return; + } + }; + commands.insert_resource(xr_session); + commands.insert_resource(xr_resolution); + commands.insert_resource(xr_format); + commands.insert_resource(xr_session_running); + commands.insert_resource(xr_frame_waiter); + commands.insert_resource(xr_swapchain); + commands.insert_resource(xr_input); + commands.insert_resource(xr_views); + commands.insert_resource(xr_frame_state); + *status = XrStatus::Enabling; + setup_xr.send_default(); +} + +fn stop_xr_session(session: ResMut, mut status: ResMut) { + match session.request_exit() { + Ok(_) => {} + Err(err) => { + error!("Error while trying to request session exit: {}", err) + } + } + *status = XrStatus::Enabling; +} diff --git a/src/xr_init/schedules.rs b/src/xr_init/schedules.rs new file mode 100644 index 0000000..f08fde6 --- /dev/null +++ b/src/xr_init/schedules.rs @@ -0,0 +1,51 @@ +use bevy::{ecs::schedule::{ScheduleLabel, Schedule, ExecutorKind}, app::App}; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPreSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPrePostSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPostSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPreCleanup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrCleanup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPostCleanup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPreRenderUpdate; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrRenderUpdate; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPostRenderUpdate; + +pub(super) fn add_schedules(app: &mut App) { + let schedules = [ + Schedule::new(XrPreSetup), + Schedule::new(XrSetup), + Schedule::new(XrPrePostSetup), + Schedule::new(XrPostSetup), + Schedule::new(XrPreRenderUpdate), + Schedule::new(XrRenderUpdate), + Schedule::new(XrPostRenderUpdate), + Schedule::new(XrPreCleanup), + Schedule::new(XrCleanup), + Schedule::new(XrPostCleanup), + ]; + for mut schedule in schedules { + schedule.set_executor_kind(ExecutorKind::SingleThreaded); + schedule.set_apply_final_deferred(true); + app.add_schedule(schedule); + } +} From db5539cfd9e5538176028dda35b23a76b54152ec Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 23 Jan 2024 06:10:01 +0100 Subject: [PATCH 17/80] removed weird file --- + | 294 -------------------------------------------------------------- 1 file changed, 294 deletions(-) delete mode 100644 + diff --git a/+ b/+ deleted file mode 100644 index 8e33f19..0000000 --- a/+ +++ /dev/null @@ -1,294 +0,0 @@ -use crate::xr_input::{QuatConv, Vec3Conv}; -use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; -use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; -use bevy::ecs::system::lifetimeless::Read; -use bevy::math::Vec3A; -use bevy::prelude::*; -use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; -use bevy::render::extract_component::ExtractComponent; -use bevy::render::primitives::Frustum; -use bevy::render::view::{ColorGrading, VisibleEntities}; -use openxr::Fovf; - -#[derive(Bundle)] -pub struct XrCamerasBundle { - pub left: XrCameraBundle, - pub right: XrCameraBundle, -} -impl XrCamerasBundle { - pub fn new() -> Self { - Self::default() - } -} -impl Default for XrCamerasBundle { - fn default() -> Self { - Self { - left: XrCameraBundle::new(Eye::Left), - right: XrCameraBundle::new(Eye::Right), - } - } -} - -#[derive(Bundle)] -pub struct XrCameraBundle { - pub camera: Camera, - pub camera_render_graph: CameraRenderGraph, - pub xr_projection: XRProjection, - pub visible_entities: VisibleEntities, - pub frustum: Frustum, - pub transform: Transform, - pub global_transform: GlobalTransform, - pub camera_3d: Camera3d, - pub tonemapping: Tonemapping, - pub dither: DebandDither, - pub color_grading: ColorGrading, - pub xr_camera_type: XrCameraType, -} -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] -pub enum XrCameraType { - Xr(Eye), - Flatscreen, -} - - -#[derive(Component)] -pub(super) struct TransformExtract; - - -impl ExtractComponent for TransformExtract { - type Query = Read; - - type Filter = (); - - type Out = Transform; - - fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { - info!("extracting Transform"); - Some(*item) - } -} - -impl ExtractComponent for XrCameraType { - type Query = Read; - - type Filter = (); - - type Out = Self; - - fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { - info!("extracting Cam Type"); - Some(*item) - } -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub enum Eye { - Left = 0, - Right = 1, -} - -impl XrCameraBundle { - pub fn new(eye: Eye) -> Self { - Self { - camera: Camera { - order: -1, - target: RenderTarget::TextureView(match eye { - Eye::Left => LEFT_XR_TEXTURE_HANDLE, - Eye::Right => RIGHT_XR_TEXTURE_HANDLE, - }), - viewport: None, - ..default() - }, - camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME), - xr_projection: Default::default(), - visible_entities: Default::default(), - frustum: Default::default(), - transform: Default::default(), - global_transform: Default::default(), - camera_3d: Default::default(), - tonemapping: Default::default(), - dither: DebandDither::Enabled, - color_grading: Default::default(), - xr_camera_type: XrCameraType::Xr(eye), - } - } -} - -#[derive(Debug, Clone, Component, Reflect, ExtractComponent)] -#[reflect(Component, Default)] -pub struct XRProjection { - pub near: f32, - pub far: f32, - #[reflect(ignore)] - pub fov: Fovf, -} - -impl Default for XRProjection { - fn default() -> Self { - Self { - near: 0.1, - far: 1000., - fov: Default::default(), - } - } -} - -impl XRProjection { - pub fn new(near: f32, far: f32, fov: Fovf) -> Self { - XRProjection { near, far, fov } - } -} - -impl CameraProjection for XRProjection { - // ============================================================================= - // math code adapted from - // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h - // Copyright (c) 2017 The Khronos Group Inc. - // Copyright (c) 2016 Oculus VR, LLC. - // SPDX-License-Identifier: Apache-2.0 - // ============================================================================= - fn get_projection_matrix(&self) -> Mat4 { - // symmetric perspective for debugging - // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); - // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); - // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); - - let fov = self.fov; - let is_vulkan_api = false; // FIXME wgpu probably abstracts this - let near_z = self.near; - let far_z = -1.; // use infinite proj - // let far_z = self.far; - - let tan_angle_left = fov.angle_left.tan(); - let tan_angle_right = fov.angle_right.tan(); - - let tan_angle_down = fov.angle_down.tan(); - let tan_angle_up = fov.angle_up.tan(); - - let tan_angle_width = tan_angle_right - tan_angle_left; - - // Set to tanAngleDown - tanAngleUp for a clip space with positive Y - // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with - // positive Y up (OpenGL / D3D / Metal). - // const float tanAngleHeight = - // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); - let tan_angle_height = if is_vulkan_api { - tan_angle_down - tan_angle_up - } else { - tan_angle_up - tan_angle_down - }; - - // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). - // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). - // const float offsetZ = - // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; - // FIXME handle enum of graphics apis - let offset_z = 0.; - - let mut cols: [f32; 16] = [0.0; 16]; - - if far_z <= near_z { - // place the far plane at infinity - cols[0] = 2. / tan_angle_width; - cols[4] = 0.; - cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; - cols[12] = 0.; - - cols[1] = 0.; - cols[5] = 2. / tan_angle_height; - cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; - cols[13] = 0.; - - cols[2] = 0.; - cols[6] = 0.; - cols[10] = -1.; - cols[14] = -(near_z + offset_z); - - cols[3] = 0.; - cols[7] = 0.; - cols[11] = -1.; - cols[15] = 0.; - - // bevy uses the _reverse_ infinite projection - // https://dev.theomader.com/depth-precision/ - let z_reversal = Mat4::from_cols_array_2d(&[ - [1f32, 0., 0., 0.], - [0., 1., 0., 0.], - [0., 0., -1., 0.], - [0., 0., 1., 1.], - ]); - - return z_reversal * Mat4::from_cols_array(&cols); - } else { - // normal projection - cols[0] = 2. / tan_angle_width; - cols[4] = 0.; - cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; - cols[12] = 0.; - - cols[1] = 0.; - cols[5] = 2. / tan_angle_height; - cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; - cols[13] = 0.; - - cols[2] = 0.; - cols[6] = 0.; - cols[10] = -(far_z + offset_z) / (far_z - near_z); - cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); - - cols[3] = 0.; - cols[7] = 0.; - cols[11] = -1.; - cols[15] = 0.; - } - - Mat4::from_cols_array(&cols) - } - - fn update(&mut self, _width: f32, _height: f32) {} - - fn far(&self) -> f32 { - self.far - } - - fn get_frustum_corners(&self, z_near: f32, z_far: f32) -> [Vec3A; 8] { - let tan_angle_left = self.fov.angle_left.tan(); - let tan_angle_right = self.fov.angle_right.tan(); - - let tan_angle_bottom = self.fov.angle_down.tan(); - let tan_angle_top = self.fov.angle_up.tan(); - - // NOTE: These vertices are in the specific order required by [`calculate_cascade`]. - [ - Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_near, // bottom right - Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_near, // top right - Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_near, // top left - Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_near, // bottom left - Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_far, // bottom right - Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_far, // top right - Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_far, // top left - Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_far, // bottom left - ] - } -} - -pub fn xr_camera_head_sync( - views: ResMut, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, -) { - //TODO calculate HMD position - for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - XrCameraType::Xr(eye) => *eye as usize, - // I don't belive we need a flatscrenn cam, that's just a cam without this component - XrCameraType::Flatscreen => continue, - }; - let view = match views.get(view_idx) { - Some(views) => views, - None => continue, - }; - xr_projection.fov = view.fov; - transform.rotation = view.pose.orientation.to_quat(); - transform.translation = view.pose.position.to_vec3(); - } -} From bc1985eeddcb33cbbb9e1174103533b0cb8178a1 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Wed, 24 Jan 2024 00:28:53 +0100 Subject: [PATCH 18/80] works on quest broken on wivrn --- src/lib.rs | 70 ++++++++++++++++++------------------------------------ 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3b52be6..3b6b4fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,6 @@ pub mod xr_input; use std::fs::File; use std::io::{BufWriter, Write}; -use std::net::TcpStream; use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; @@ -21,9 +20,10 @@ use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; -use bevy::render::renderer::RenderInstance; +use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; +use bevy::transform::systems::{propagate_transforms, sync_simple_transforms}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; @@ -142,16 +142,25 @@ impl Plugin for OpenXrPlugin { xr_pre_frame .run_if(xr_only()) .run_if(xr_render_only()) - .in_set(RenderSet::Prepare), - ); - render_app.add_systems( - Render, - (locate_views, xr_input::xr_camera::xr_camera_head_sync) - .chain() - .run_if(xr_only()) - .run_if(xr_render_only()) - .in_set(RenderSet::Prepare), + // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet + // right before rendering + .before(render_system) + .after(RenderSet::ExtractCommands), + // .in_set(RenderSet::Prepare), ); + // render_app.add_systems( + // Render, + // ( + // locate_views, + // xr_input::xr_camera::xr_camera_head_sync, + // sync_simple_transforms, + // propagate_transforms, + // ) + // .chain() + // .run_if(xr_only()) + // .run_if(xr_render_only()) + // .in_set(RenderSet::Prepare), + // ); render_app.add_systems( Render, xr_end_frame @@ -159,15 +168,9 @@ impl Plugin for OpenXrPlugin { .run_if(xr_render_only()) .after(RenderSet::Render), ); - render_app.insert_resource(TcpConnection( - TcpStream::connect("192.168.2.100:6969").unwrap(), - )); } } -#[derive(Resource)] -struct TcpConnection(TcpStream); - #[derive(Default)] pub struct DefaultXrPlugins { pub reqeusted_extensions: XrExtensions, @@ -180,7 +183,7 @@ impl PluginGroup for DefaultXrPlugins { DefaultPlugins .build() .disable::() - .disable::() + // .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, reqeusted_extensions: self.reqeusted_extensions, @@ -273,6 +276,7 @@ pub fn xr_wait_frame( ) { { let _span = info_span!("xr_wait_frame").entered(); + info!("Pre Frame Wait"); *frame_state = match frame_waiter.wait() { Ok(a) => a.into(), Err(e) => { @@ -280,6 +284,7 @@ pub fn xr_wait_frame( return; } }; + info!("Post Frame Wait"); **should_render = frame_state.should_render; } } @@ -296,9 +301,7 @@ pub fn xr_pre_frame( } { let _span = info_span!("xr_wait_image").entered(); - info!("wait image"); swapchain.wait_image().unwrap(); - info!("waited image"); } { let _span = info_span!("xr_update_manual_texture_views").entered(); @@ -325,8 +328,6 @@ pub fn xr_end_frame( swapchain: Res, resolution: Res, environment_blend_mode: Res, - mut connection: ResMut, - cams: Query<(&Transform, &XrCameraType)>, ) { #[cfg(target_os = "android")] { @@ -338,31 +339,6 @@ pub fn xr_end_frame( let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); } - let mut cam = None; - for (t, c) in &cams { - if *c == XrCameraType::Xr(xr_input::xr_camera::Eye::Left) { - cam = Some(*t); - break; - } - } - let _ = std::writeln!( - &mut connection.0, - "{},{},{},{},{},{},{},{},{},{},{},{},{},{}", - views[0].pose.position.x, - views[0].pose.position.y, - views[0].pose.position.z, - views[0].pose.orientation.x, - views[0].pose.orientation.y, - views[0].pose.orientation.z, - views[0].pose.orientation.w, - cam.unwrap().translation.x, - cam.unwrap().translation.y, - cam.unwrap().translation.z, - cam.unwrap().rotation.x, - cam.unwrap().rotation.y, - cam.unwrap().rotation.z, - cam.unwrap().rotation.w, - ); { let _span = info_span!("xr_end_frame").entered(); let result = swapchain.end( From a3c33cb8b6928ebc9990c1b711cf3086e2264b39 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Wed, 24 Jan 2024 03:46:20 +0100 Subject: [PATCH 19/80] win + steamvr and quest working, wivrn still broken --- src/lib.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3b6b4fe..f5675b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,7 @@ use input::XrInput; use openxr as xr; // use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; use resources::*; -use xr::FormFactor; +use xr::{FormFactor, FrameState}; use xr_init::{ xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, }; @@ -119,9 +119,7 @@ impl Plugin for OpenXrPlugin { PreUpdate, ( xr_reset_should_render, - apply_deferred, xr_wait_frame.run_if(xr_only()), - apply_deferred, locate_views.run_if(xr_only()), apply_deferred, ) @@ -146,7 +144,7 @@ impl Plugin for OpenXrPlugin { // right before rendering .before(render_system) .after(RenderSet::ExtractCommands), - // .in_set(RenderSet::Prepare), + // .in_set(RenderSet::Prepare), ); // render_app.add_systems( // Render, @@ -284,6 +282,13 @@ pub fn xr_wait_frame( return; } }; + #[allow(clippy::erasing_op)] + { + frame_state.predicted_display_time = xr::Time::from_nanos( + frame_state.predicted_display_time.as_nanos() + + (frame_state.predicted_display_period.as_nanos() * 1), + ); + }; info!("Post Frame Wait"); **should_render = frame_state.should_render; } From 9d6d60cbe73976e7ab08ea344167e7205e1711fc Mon Sep 17 00:00:00 2001 From: Schmarni Date: Thu, 25 Jan 2024 06:39:44 +0100 Subject: [PATCH 20/80] implement simple pipelined rendering --- src/lib.rs | 45 ++++++++++++++++++++++++++++++++++++--------- src/xr_init/mod.rs | 17 +++++++++++++---- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f5675b7..2659469 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ use openxr as xr; use resources::*; use xr::{FormFactor, FrameState}; use xr_init::{ - xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, + xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, XrHasWaited, xr_after_wait_only, }; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; @@ -118,7 +118,7 @@ impl Plugin for OpenXrPlugin { app.add_systems( PreUpdate, ( - xr_reset_should_render, + xr_reset_per_frame_resources, xr_wait_frame.run_if(xr_only()), locate_views.run_if(xr_only()), apply_deferred, @@ -130,15 +130,15 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_begin_frame - .run_if(xr_only()) - .run_if(xr_render_only()) + .run_if(xr_only()).run_if(xr_after_wait_only()) + // .run_if(xr_render_only()) .after(RenderSet::ExtractCommands) .before(xr_pre_frame), ); render_app.add_systems( Render, xr_pre_frame - .run_if(xr_only()) + .run_if(xr_only()).run_if(xr_after_wait_only()) .run_if(xr_render_only()) // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet // right before rendering @@ -162,10 +162,34 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_end_frame - .run_if(xr_only()) + .run_if(xr_only()).run_if(xr_after_wait_only()) .run_if(xr_render_only()) .after(RenderSet::Render), ); + render_app.add_systems( + Render, + xr_skip_frame + .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(not(xr_render_only())) + .after(RenderSet::Render), + ); + } +} + +fn xr_skip_frame( + xr_swapchain: Res, + xr_frame_state: Res, + environment_blend_mode: Res, +) { + let swapchain: &Swapchain = &xr_swapchain; + // swapchain.begin().unwrap(); + match swapchain { + Swapchain::Vulkan(swap) => { + swap.stream + .lock() + .unwrap() + .end(xr_frame_state.predicted_display_time, **environment_blend_mode, &[]).unwrap(); + } } } @@ -180,8 +204,8 @@ impl PluginGroup for DefaultXrPlugins { fn build(self) -> PluginGroupBuilder { DefaultPlugins .build() + .disable::() .disable::() - // .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, reqeusted_extensions: self.reqeusted_extensions, @@ -211,8 +235,9 @@ impl PluginGroup for DefaultXrPlugins { } } -fn xr_reset_should_render(mut should: ResMut) { +fn xr_reset_per_frame_resources(mut should: ResMut,mut waited: ResMut) { **should = false; + **waited = false; } fn xr_poll_events( @@ -271,6 +296,7 @@ pub fn xr_wait_frame( mut frame_state: ResMut, mut frame_waiter: ResMut, mut should_render: ResMut, + mut waited: ResMut, ) { { let _span = info_span!("xr_wait_frame").entered(); @@ -286,11 +312,12 @@ pub fn xr_wait_frame( { frame_state.predicted_display_time = xr::Time::from_nanos( frame_state.predicted_display_time.as_nanos() - + (frame_state.predicted_display_period.as_nanos() * 1), + + (frame_state.predicted_display_period.as_nanos() * 0), ); }; info!("Post Frame Wait"); **should_render = frame_state.should_render; + **waited = true; } } diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs index d7db1c9..06d9578 100644 --- a/src/xr_init/mod.rs +++ b/src/xr_init/mod.rs @@ -33,16 +33,23 @@ pub enum XrStatus { Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut, )] pub struct XrShouldRender(bool); +#[derive( + Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut, +)] +pub struct XrHasWaited(bool); pub struct XrEarlyInitPlugin; pub struct XrInitPlugin; -pub fn xr_only() -> impl FnMut(Option>) -> bool { - resource_exists_and_equals(XrStatus::Enabled) +pub fn xr_only() -> impl FnMut(Res) -> bool { + resource_equals(XrStatus::Enabled) } -pub fn xr_render_only() -> impl FnMut(Option>) -> bool { - resource_exists_and_equals(XrShouldRender(true)) +pub fn xr_render_only() -> impl FnMut(Res) -> bool { + resource_equals(XrShouldRender(true)) +} +pub fn xr_after_wait_only() -> impl FnMut(Res) -> bool { + resource_equals(XrHasWaited(true)) } impl Plugin for XrEarlyInitPlugin { @@ -59,7 +66,9 @@ impl Plugin for XrInitPlugin { add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); app.init_resource::(); + app.init_resource::(); app.add_systems(PreUpdate, setup_xr.run_if(on_event::())) .add_systems(PreUpdate, cleanup_xr.run_if(on_event::())); app.add_systems( From ac7c1089d954cc60c48652aae25dfacaeaf1e029 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Fri, 26 Jan 2024 04:35:51 +0100 Subject: [PATCH 21/80] fixed pipelined rendering on quest 2 --- src/lib.rs | 81 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2659469..8d77c30 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ use crate::xr_init::{StartXrSession, XrInitPlugin}; use crate::xr_input::hands::hand_tracking::DisableHandTracking; use crate::xr_input::oculus_touch::ActionSets; use bevy::app::{AppExit, PluginGroupBuilder}; +use bevy::core::TaskPoolThreadAssignmentPolicy; use bevy::ecs::system::SystemState; use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; @@ -23,6 +24,7 @@ use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; +use bevy::tasks::available_parallelism; use bevy::transform::systems::{propagate_transforms, sync_simple_transforms}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; use graphics::extensions::XrExtensions; @@ -33,7 +35,8 @@ use openxr as xr; use resources::*; use xr::{FormFactor, FrameState}; use xr_init::{ - xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, XrHasWaited, xr_after_wait_only, + xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrHasWaited, + XrShouldRender, XrStatus, }; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; @@ -130,7 +133,8 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_begin_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) // .run_if(xr_render_only()) .after(RenderSet::ExtractCommands) .before(xr_pre_frame), @@ -138,7 +142,8 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_pre_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) .run_if(xr_render_only()) // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet // right before rendering @@ -146,30 +151,32 @@ impl Plugin for OpenXrPlugin { .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), ); - // render_app.add_systems( - // Render, - // ( - // locate_views, - // xr_input::xr_camera::xr_camera_head_sync, - // sync_simple_transforms, - // propagate_transforms, - // ) - // .chain() - // .run_if(xr_only()) - // .run_if(xr_render_only()) - // .in_set(RenderSet::Prepare), - // ); + render_app.add_systems( + Render, + ( + locate_views, + xr_input::xr_camera::xr_camera_head_sync, + sync_simple_transforms, + propagate_transforms, + ) + .chain() + .run_if(xr_only()) + // .run_if(xr_render_only()) + .in_set(RenderSet::Prepare), + ); render_app.add_systems( Render, xr_end_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) .run_if(xr_render_only()) .after(RenderSet::Render), ); render_app.add_systems( Render, xr_skip_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) .run_if(not(xr_render_only())) .after(RenderSet::Render), ); @@ -188,7 +195,12 @@ fn xr_skip_frame( swap.stream .lock() .unwrap() - .end(xr_frame_state.predicted_display_time, **environment_blend_mode, &[]).unwrap(); + .end( + xr_frame_state.predicted_display_time, + **environment_blend_mode, + &[], + ) + .unwrap(); } } } @@ -204,7 +216,20 @@ impl PluginGroup for DefaultXrPlugins { fn build(self) -> PluginGroupBuilder { DefaultPlugins .build() - .disable::() + .set(TaskPoolPlugin { + task_pool_options: TaskPoolOptions { + compute: TaskPoolThreadAssignmentPolicy { + // set the minimum # of compute threads + // to the total number of available threads + min_threads: 2, + max_threads: std::usize::MAX, // unlimited max threads + percent: 1.0, // this value is irrelevant in this case + }, + // keep the defaults for everything else + ..default() + }, + }) + // .disable::() .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, @@ -235,7 +260,10 @@ impl PluginGroup for DefaultXrPlugins { } } -fn xr_reset_per_frame_resources(mut should: ResMut,mut waited: ResMut) { +fn xr_reset_per_frame_resources( + mut should: ResMut, + mut waited: ResMut, +) { **should = false; **waited = false; } @@ -308,13 +336,10 @@ pub fn xr_wait_frame( return; } }; - #[allow(clippy::erasing_op)] - { - frame_state.predicted_display_time = xr::Time::from_nanos( - frame_state.predicted_display_time.as_nanos() - + (frame_state.predicted_display_period.as_nanos() * 0), - ); - }; + frame_state.predicted_display_time = xr::Time::from_nanos( + frame_state.predicted_display_time.as_nanos() + + frame_state.predicted_display_period.as_nanos(), + ); info!("Post Frame Wait"); **should_render = frame_state.should_render; **waited = true; From 1e4db18f84f35bc2c22ef2d7337dc1cef6abb46c Mon Sep 17 00:00:00 2001 From: Schmarni Date: Sat, 27 Jan 2024 04:06:27 +0100 Subject: [PATCH 22/80] commit --- src/lib.rs | 12 ++++++------ src/xr_input/xr_camera.rs | 16 +--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8d77c30..44fc206 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,9 @@ use bevy::app::{AppExit, PluginGroupBuilder}; use bevy::core::TaskPoolThreadAssignmentPolicy; use bevy::ecs::system::SystemState; use bevy::prelude::*; -use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; +use bevy::render::camera::{ + camera_system, ManualTextureView, ManualTextureViewHandle, ManualTextureViews, +}; use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; @@ -42,7 +44,7 @@ use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin}; use xr_input::hands::XrHandPlugins; -use xr_input::xr_camera::XrCameraType; +use xr_input::xr_camera::{XRProjection, XrCameraType}; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -170,7 +172,7 @@ impl Plugin for OpenXrPlugin { .run_if(xr_only()) .run_if(xr_after_wait_only()) .run_if(xr_render_only()) - .after(RenderSet::Render), + .in_set(RenderSet::Cleanup), ); render_app.add_systems( Render, @@ -178,7 +180,7 @@ impl Plugin for OpenXrPlugin { .run_if(xr_only()) .run_if(xr_after_wait_only()) .run_if(not(xr_render_only())) - .after(RenderSet::Render), + .in_set(RenderSet::Cleanup), ); } } @@ -219,8 +221,6 @@ impl PluginGroup for DefaultXrPlugins { .set(TaskPoolPlugin { task_pool_options: TaskPoolOptions { compute: TaskPoolThreadAssignmentPolicy { - // set the minimum # of compute threads - // to the total number of available threads min_threads: 2, max_threads: std::usize::MAX, // unlimited max threads percent: 1.0, // this value is irrelevant in this case diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 1d143c3..95b70b1 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -44,17 +44,15 @@ pub struct XrCameraBundle { pub color_grading: ColorGrading, pub xr_camera_type: XrCameraType, } -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)] pub enum XrCameraType { Xr(Eye), Flatscreen, } - #[derive(Component)] pub(super) struct TransformExtract; - impl ExtractComponent for TransformExtract { type Query = Read; @@ -67,18 +65,6 @@ impl ExtractComponent for TransformExtract { } } -impl ExtractComponent for XrCameraType { - type Query = Read; - - type Filter = (); - - type Out = Self; - - fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { - Some(*item) - } -} - #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, From 5c81f135e792a79493c0cc279bfda50331fd48fc Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 29 Jan 2024 04:17:20 +0100 Subject: [PATCH 23/80] fix late latching? --- src/lib.rs | 8 ++++++++ src/xr_input/xr_camera.rs | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 44fc206..80382aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; +use bevy::render::view::ExtractedView; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::tasks::available_parallelism; use bevy::transform::systems::{propagate_transforms, sync_simple_transforms}; @@ -160,6 +161,7 @@ impl Plugin for OpenXrPlugin { xr_input::xr_camera::xr_camera_head_sync, sync_simple_transforms, propagate_transforms, + update_cam_views, ) .chain() .run_if(xr_only()) @@ -185,6 +187,12 @@ impl Plugin for OpenXrPlugin { } } +fn update_cam_views(mut query: Query<(&mut ExtractedView, &GlobalTransform)>) { + for (mut view, transform) in &mut query { + view.transform = *transform; + } +} + fn xr_skip_frame( xr_swapchain: Res, xr_frame_state: Res, diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 95b70b1..912b3ec 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -65,6 +65,20 @@ impl ExtractComponent for TransformExtract { } } +#[derive(Component)] +pub(super) struct GlobalTransformExtract; + +impl ExtractComponent for GlobalTransformExtract { + type Query = Read; + + type Filter = (); + + type Out = GlobalTransform; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + Some(*item) + } +} #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, From 71a08798ef9f6db71cac672b570ede49c245bf79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Hogsl=C3=A4tt?= <74669087+RasmusHogslatt@users.noreply.github.com> Date: Thu, 1 Feb 2024 05:05:24 +0100 Subject: [PATCH 24/80] Working passthrough for Meta Quest 3 (#66) * Window is None * Builds but check manifest * debug prints * Started, not passing last "cvt" * Passthrough works, bevy not visible * Passthrough working * Passthrough working * Working passthrough --- examples/android/manifest.yaml | 12 ++- examples/android/src/lib.rs | 29 ++++-- src/graphics/extensions.rs | 16 +-- src/graphics/mod.rs | 5 +- src/graphics/vulkan.rs | 20 +++- src/lib.rs | 111 +++++++++++++------- src/passthrough.rs | 179 +++++++++++++++++++++++++++++++++ src/resources.rs | 155 ++++++++++++++-------------- src/xr_init.rs | 8 +- 9 files changed, 398 insertions(+), 137 deletions(-) create mode 100644 src/passthrough.rs diff --git a/examples/android/manifest.yaml b/examples/android/manifest.yaml index 9a6118c..7622de3 100644 --- a/examples/android/manifest.yaml +++ b/examples/android/manifest.yaml @@ -3,6 +3,15 @@ android: - "runtime_libs" manifest: package: "org.bevyengine.example_openxr_android" + uses_feature: + - name: "android.hardware.vr.headtracking" + required: true + - name: "oculus.software.handtracking" + required: true + - name: "com.oculus.feature.PASSTHROUGH" + required: true + - name: "com.oculus.experimental.enabled" + required: true application: label: "Bevy Openxr Android" theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen" @@ -14,7 +23,7 @@ android: - name: "com.oculus.supportedDevices" value: "quest|quest2|quest3|questpro" activities: - - config_changes: "density|keyboard|keyboardHidden|navigation|orientation|screenLayout|screenSize|uiMode" + - config_changes: "density|keyboard|keyboardHidden|navigation|orientation|screenLayout|screenSize|uiMode|screenLayout" launch_mode: "singleTask" orientation: "landscape" intent_filters: @@ -23,5 +32,6 @@ android: categories: - "com.oculus.intent.category.VR" - "android.intent.category.LAUNCHER" + - "org.khronos.openxr.intent.category.IMMERSIVE_HMD" sdk: target_sdk_version: 32 diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index 4fabb2b..efb835a 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -1,7 +1,11 @@ use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}; use bevy::prelude::*; use bevy::transform::components::Transform; +use bevy_oxr::graphics::extensions::XrExtensions; use bevy_oxr::graphics::XrAppInfo; +use bevy_oxr::graphics::XrPreferdBlendMode::AlphaBlend; +use bevy_oxr::passthrough::{passthrough_layer_pause, passthrough_layer_resume}; +use bevy_oxr::xr_init::XrRenderData; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ @@ -11,18 +15,21 @@ use bevy_oxr::DefaultXrPlugins; #[bevy_main] fn main() { + let mut xr_extensions = XrExtensions::default(); + xr_extensions.enable_fb_passthrough(); App::new() .add_plugins(DefaultXrPlugins { + reqeusted_extensions: xr_extensions, app_info: XrAppInfo { name: "Bevy OXR Android Example".into(), }, - ..default() + prefered_blend_mode: bevy_oxr::graphics::XrPreferdBlendMode::Opaque, }) .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) .add_systems(Startup, setup) - .add_systems(Update, proto_locomotion) + .add_systems(Update, (proto_locomotion, toggle_passthrough)) .add_systems(Startup, spawn_controllers_example) .insert_resource(PrototypeLocomotionConfig::default()) .run(); @@ -64,11 +71,6 @@ fn setup( transform: Transform::from_xyz(4.0, 8.0, 4.0), ..default() }); - // camera - // commands.spawn((Camera3dBundle { - // transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y), - // ..default() - // },)); } fn spawn_controllers_example(mut commands: Commands) { @@ -87,3 +89,16 @@ fn spawn_controllers_example(mut commands: Commands) { SpatialBundle::default(), )); } + +// Does this work? Not getting logs +fn toggle_passthrough(keys: Res>, mut xr_data: ResMut) { + if keys.just_pressed(KeyCode::Space) { + if xr_data.xr_passthrough_active { + passthrough_layer_pause(xr_data); + bevy::log::info!("Passthrough paused"); + } else { + passthrough_layer_resume(xr_data); + bevy::log::info!("Passthrough resumed"); + } + } +} diff --git a/src/graphics/extensions.rs b/src/graphics/extensions.rs index 2276b79..81d8ca7 100644 --- a/src/graphics/extensions.rs +++ b/src/graphics/extensions.rs @@ -10,14 +10,14 @@ impl XrExtensions { pub fn raw(&self) -> &ExtensionSet { &self.0 } - // pub fn enable_fb_passthrough(&mut self) -> &mut Self { - // self.0.fb_passthrough = true; - // self - // } - // pub fn disable_fb_passthrough(&mut self) -> &mut Self { - // self.0.fb_passthrough = false; - // self - // } + pub fn enable_fb_passthrough(&mut self) -> &mut Self { + self.0.fb_passthrough = true; + self + } + pub fn disable_fb_passthrough(&mut self) -> &mut Self { + self.0.fb_passthrough = false; + self + } pub fn enable_hand_tracking(&mut self) -> &mut Self { self.0.ext_hand_tracking = true; self diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 0aa3dfc..6445986 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -6,9 +6,10 @@ use bevy::window::RawHandleWrapper; use wgpu::Instance; use crate::input::XrInput; +use crate::passthrough::{Passthrough, PassthroughLayer}; use crate::resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrPassthrough, + XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }; use openxr as xr; diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index ba9814e..2678079 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -14,9 +14,12 @@ use xr::EnvironmentBlendMode; use crate::graphics::extensions::XrExtensions; use crate::input::XrInput; + +use crate::passthrough::{Passthrough, PassthroughLayer}; use crate::resources::{ Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, - XrInstance, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrInstance, XrPassthrough, XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, + XrSwapchain, XrViews, }; use crate::VIEW_TYPE; @@ -54,7 +57,7 @@ pub fn initialize_xr_graphics( let available_extensions: XrExtensions = xr_entry.enumerate_extensions()?.into(); assert!(available_extensions.raw().khr_vulkan_enable2); - info!("available xr exts: {:#?}", available_extensions); + //info!("available xr exts: {:#?}", available_extensions); let mut enabled_extensions: xr::ExtensionSet = (available_extensions & reqeusted_extensions).into(); @@ -65,7 +68,7 @@ pub fn initialize_xr_graphics( } let available_layers = xr_entry.enumerate_layers()?; - info!("available xr layers: {:#?}", available_layers); + //info!("available xr layers: {:#?}", available_layers); let xr_instance = xr_entry.create_instance( &xr::ApplicationInfo { @@ -95,20 +98,25 @@ pub fn initialize_xr_graphics( let blend_modes = xr_instance.enumerate_environment_blend_modes(xr_system_id, VIEW_TYPE)?; let blend_mode: EnvironmentBlendMode = match prefered_blend_mode { XrPreferdBlendMode::Opaque if blend_modes.contains(&EnvironmentBlendMode::OPAQUE) => { + bevy::log::info!("Using Opaque"); EnvironmentBlendMode::OPAQUE } XrPreferdBlendMode::Additive if blend_modes.contains(&EnvironmentBlendMode::ADDITIVE) => { + bevy::log::info!("Using Additive"); EnvironmentBlendMode::ADDITIVE } XrPreferdBlendMode::AlphaBlend if blend_modes.contains(&EnvironmentBlendMode::ALPHA_BLEND) => { + bevy::log::info!("Using AlphaBlend"); EnvironmentBlendMode::ALPHA_BLEND } - _ => EnvironmentBlendMode::OPAQUE, + _ => { + bevy::log::info!("Using Opaque"); + EnvironmentBlendMode::OPAQUE + } }; - #[cfg(not(target_os = "android"))] let vk_target_version = vk::make_api_version(0, 1, 2, 0); #[cfg(not(target_os = "android"))] @@ -324,6 +332,8 @@ pub fn initialize_xr_graphics( .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0]) .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb); + // TODO: Log swapchain format + let resolution = uvec2( views[0].recommended_image_rect_width, views[0].recommended_image_rect_height, diff --git a/src/lib.rs b/src/lib.rs index fed5e5b..8a96292 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ pub mod graphics; pub mod input; -// pub mod passthrough; +pub mod passthrough; pub mod resource_macros; pub mod resources; pub mod xr_init; @@ -24,7 +24,7 @@ use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; use input::XrInput; use openxr as xr; -// use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; +use passthrough::{start_passthrough, supports_passthrough, Passthrough, PassthroughLayer}; use resources::*; use xr::FormFactor; use xr_init::{xr_only, XrEnableStatus, XrRenderData}; @@ -62,13 +62,16 @@ pub struct FutureXrResources( XrInput, XrViews, XrFrameState, + bool, + XrPassthrough, + XrPassthroughLayer, )>, >, >, ); -// fn mr_test(mut commands: Commands, passthrough_layer: Option>) { -// commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); -// } +fn mr_test(mut commands: Commands, passthrough_layer: Option>) { + commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); +} impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { @@ -115,20 +118,60 @@ impl Plugin for OpenXrPlugin { app.insert_resource(input.clone()); app.insert_resource(views.clone()); app.insert_resource(frame_state.clone()); - let xr_data = XrRenderData { - xr_instance, - xr_session: session, - xr_blend_mode: blend_mode, - xr_resolution: resolution, - xr_format: format, - xr_session_running: session_running, - xr_frame_waiter: frame_waiter, - xr_swapchain: swapchain, - xr_input: input, - xr_views: views, - xr_frame_state: frame_state, - }; - app.insert_resource(xr_data); + + // Check if the fb_passthrough extension is available + let fb_passthrough_available = xr_instance.exts().fb_passthrough.is_some(); + bevy::log::info!( + "From OpenXrPlugin: fb_passthrough_available: {}", + fb_passthrough_available + ); + // Get the system for the head-mounted display + let hmd_system = xr_instance + .system(FormFactor::HEAD_MOUNTED_DISPLAY) + .unwrap(); + bevy::log::info!("From OpenXrPlugin: hmd_system: {:?}", hmd_system); + + // Check if the system supports passthrough + let passthrough_supported = + supports_passthrough(&xr_instance, hmd_system).is_ok_and(|v| v); + bevy::log::info!( + "From OpenXrPlugin: passthrough_supported: {}", + passthrough_supported + ); + + // The passthrough variable will be true only if both fb_passthrough is available and the system supports passthrough + let passthrough = fb_passthrough_available && passthrough_supported; + bevy::log::info!("From OpenXrPlugin: passthrough: {}", passthrough); + + let mut p: Option = None; + let mut pl: Option = None; + if passthrough { + if let Ok((p, pl)) = start_passthrough(&xr_instance, &session) { + let xr_data = XrRenderData { + xr_instance, + xr_session: session, + xr_blend_mode: blend_mode, + xr_resolution: resolution, + xr_format: format, + xr_session_running: session_running, + xr_frame_waiter: frame_waiter, + xr_swapchain: swapchain, + xr_input: input, + xr_views: views, + xr_frame_state: frame_state, + xr_passthrough_active: true, + xr_passthrough: XrPassthrough::new(Mutex::new(p)), + xr_passthrough_layer: XrPassthroughLayer::new(Mutex::new(pl)), + }; + bevy::log::info!("Passthrough is supported!"); + app.insert_resource(xr_data); + app.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); + } + + if !app.world.contains_resource::() { + info!("ClearColor!"); + } + } app.insert_resource(ActionSets(vec![])); app.add_plugins(RenderPlugin { render_creation: RenderCreation::Manual( @@ -179,23 +222,6 @@ impl Plugin for OpenXrPlugin { } else { app.insert_resource(DisableHandTracking::Both); } - // let passthrough = data.xr_instance.exts().fb_passthrough.is_some() - // && supports_passthrough( - // &data.xr_instance, - // data.xr_instance - // .system(FormFactor::HEAD_MOUNTED_DISPLAY) - // .unwrap(), - // ) - // .is_ok_and(|v| v); - // if passthrough { - // info!("Passthrough!"); - // let (pl, p) = start_passthrough(&data); - // app.insert_resource(pl); - // app.insert_resource(p); - // // if !app.world.contains_resource::() { - // // info!("ClearColor!"); - // // } - // } let (left, right) = data.xr_swapchain.get_render_views(); let left = ManualTextureView { @@ -226,6 +252,8 @@ impl Plugin for OpenXrPlugin { render_app.insert_resource(data.xr_input.clone()); render_app.insert_resource(data.xr_views.clone()); render_app.insert_resource(data.xr_frame_state.clone()); + render_app.insert_resource(data.xr_passthrough.clone()); + render_app.insert_resource(data.xr_passthrough_layer.clone()); render_app.insert_resource(XrEnableStatus::Enabled); render_app.add_systems( Render, @@ -272,7 +300,7 @@ impl PluginGroup for DefaultXrPlugins { ..default() }), #[cfg(target_os = "android")] - primary_window: None, + primary_window: None, // ? #[cfg(target_os = "android")] exit_condition: bevy::window::ExitCondition::DontExit, #[cfg(target_os = "android")] @@ -393,7 +421,7 @@ pub fn end_frame( swapchain: Res, resolution: Res, environment_blend_mode: Res, - // passthrough_layer: Option>, + passthrough_layer: Option>, ) { #[cfg(target_os = "android")] { @@ -408,13 +436,18 @@ pub fn end_frame( } { let _span = info_span!("xr_end_frame").entered(); + // bevy::log::info!( + // "passthrough_layer.is_some(): {:?}", + // passthrough_layer.is_some() + // ); + let result = swapchain.end( xr_frame_state.lock().unwrap().predicted_display_time, &views.lock().unwrap(), &input.stage, **resolution, **environment_blend_mode, - // passthrough_layer.map(|p| p.into_inner()), + passthrough_layer.map(|p| PassthroughLayer(*p.lock().unwrap())), ); match result { Ok(_) => {} diff --git a/src/passthrough.rs b/src/passthrough.rs new file mode 100644 index 0000000..26650e1 --- /dev/null +++ b/src/passthrough.rs @@ -0,0 +1,179 @@ +use bevy::prelude::*; +use std::{marker::PhantomData, mem, ptr}; + +use crate::xr_init::XrRenderData; +use openxr as xr; +use xr::{ + sys::{ + PassthroughCreateInfoFB, PassthroughFB, PassthroughLayerFB, Space, + SystemPassthroughProperties2FB, + }, + CompositionLayerBase, CompositionLayerFlags, Graphics, PassthroughCapabilityFlagsFB, +}; + +use crate::resources::XrInstance; +use crate::resources::XrSession; +pub struct PassthroughLayer(pub xr::sys::PassthroughLayerFB); +pub struct Passthrough(pub xr::sys::PassthroughFB); +fn cvt(x: xr::sys::Result) -> xr::Result { + if x.into_raw() >= 0 { + Ok(x) + } else { + Err(x) + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub(crate) struct CompositionLayerPassthrough<'a, G: xr::Graphics> { + inner: xr::sys::CompositionLayerPassthroughFB, + _marker: PhantomData<&'a G>, +} +impl<'a, G: Graphics> std::ops::Deref for CompositionLayerPassthrough<'a, G> { + type Target = CompositionLayerBase<'a, G>; + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { mem::transmute(&self.inner) } + } +} + +impl<'a, G: xr::Graphics> CompositionLayerPassthrough<'a, G> { + pub(crate) fn from_xr_passthrough_layer(layer: &PassthroughLayer) -> Self { + Self { + inner: xr::sys::CompositionLayerPassthroughFB { + ty: xr::sys::CompositionLayerPassthroughFB::TYPE, + next: ptr::null(), + flags: CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA, + space: Space::NULL, + layer_handle: layer.0, + }, + _marker: PhantomData, + } + } +} +#[inline] +pub fn supports_passthrough(instance: &XrInstance, system: xr::SystemId) -> xr::Result { + unsafe { + let mut hand = xr::sys::SystemPassthroughProperties2FB { + ty: SystemPassthroughProperties2FB::TYPE, + next: ptr::null(), + capabilities: PassthroughCapabilityFlagsFB::PASSTHROUGH_CAPABILITY, + }; + let mut p = xr::sys::SystemProperties::out(&mut hand as *mut _ as _); + cvt((instance.fp().get_system_properties)( + instance.as_raw(), + system, + p.as_mut_ptr(), + ))?; + bevy::log::info!( + "From supports_passthrough: Passthrough capabilities: {:?}", + hand.capabilities + ); + Ok( + (hand.capabilities & PassthroughCapabilityFlagsFB::PASSTHROUGH_CAPABILITY) + == PassthroughCapabilityFlagsFB::PASSTHROUGH_CAPABILITY, + ) + } +} + +#[inline] +pub fn start_passthrough( + instance: &XrInstance, + xr_session: &XrSession, +) -> xr::Result<(xr::sys::PassthroughFB, xr::sys::PassthroughLayerFB)> { + unsafe { + // Create feature + let mut passthrough_feature = xr::sys::PassthroughFB::NULL; + let mut passthrough_create_info = xr::sys::PassthroughCreateInfoFB { + ty: xr::sys::StructureType::PASSTHROUGH_CREATE_INFO_FB, // XR_TYPE_PASSTHROUGH_CREATE_INFO_FB + next: ptr::null(), + flags: xr::sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION, + }; + // bevy::log::info!("xr_session.as_raw(): {:?}", xr_session.as_raw()); + // bevy::log::info!("&passthrough_create_info: {:?}", &passthrough_create_info); + // bevy::log::info!("&mut passthrough_feature: {:?}", &mut passthrough_feature); + // bevy::log::info!( + // "instance.exts().fb_passthrough.unwrap(): {:?}", + // instance.exts().fb_passthrough.is_some() + // ); + cvt( + (instance.exts().fb_passthrough.unwrap().create_passthrough)( + xr_session.as_raw(), + &passthrough_create_info as *const _, + &mut passthrough_feature as *mut _, + ), + )?; + // bevy::log::info!("Created passthrough feature"); + // Create layer + let mut passthrough_layer = xr::sys::PassthroughLayerFB::NULL; + let mut layer_create_info: xr::sys::PassthroughLayerCreateInfoFB = + xr::sys::PassthroughLayerCreateInfoFB { + ty: xr::sys::StructureType::PASSTHROUGH_LAYER_CREATE_INFO_FB, // XR_TYPE_PASSTHROUGH_LAYER_CREATE_INFO_FB + next: ptr::null(), + passthrough: passthrough_feature, // XR_PASSTHROUGH_HANDLE + flags: xr::sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION, // XR_PASSTHROUGH_IS_RUNNING_AT_CREATION_BIT_FB + purpose: xr::sys::PassthroughLayerPurposeFB::RECONSTRUCTION, // XR_PASSTHROUGH_LAYER_PURPOSE_RECONSTRUCTION_FB + }; + cvt((instance + .exts() + .fb_passthrough + .unwrap() + .create_passthrough_layer)( + xr_session.as_raw(), + &layer_create_info as *const _, + &mut passthrough_layer as *mut _, + ))?; + // bevy::log::info!("Created passthrough layer"); + // // Start layer + + // bevy::log::info!("passthrough_feature: {:?}", passthrough_feature); + // // cvt((instance.exts().fb_passthrough.unwrap().passthrough_start)( + // // passthrough_feature, + // // ))?; + // bevy::log::info!("Started passthrough layer"); + // bevy::log::info!("Passed everything in start"); + Ok((passthrough_feature, passthrough_layer)) + } +} + +#[inline] +pub fn passthrough_layer_resume(mut xr_data_resource: ResMut) -> xr::Result<()> { + unsafe { + let passthrough_layer = &xr_data_resource.xr_passthrough_layer; + { + let passthrough_layer_locked = passthrough_layer.lock().unwrap(); + cvt((xr_data_resource + .xr_instance + .exts() + .fb_passthrough + .unwrap() + .passthrough_layer_resume)( + *passthrough_layer_locked + ))?; + } + xr_data_resource.xr_passthrough_active = true; + bevy::log::info!("Resumed passthrough layer"); + Ok(()) + } +} + +#[inline] +pub fn passthrough_layer_pause(mut xr_data_resource: ResMut) -> xr::Result<()> { + unsafe { + let passthrough_layer = &xr_data_resource.xr_passthrough_layer; + { + let passthrough_layer_locked = passthrough_layer.lock().unwrap(); + cvt((xr_data_resource + .xr_instance + .exts() + .fb_passthrough + .unwrap() + .passthrough_layer_pause)( + *passthrough_layer_locked + ))?; + } + xr_data_resource.xr_passthrough_active = false; + bevy::log::info!("Paused passthrough layer"); + Ok(()) + } +} diff --git a/src/resources.rs b/src/resources.rs index a798955..af8fb66 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -1,9 +1,12 @@ use std::sync::atomic::AtomicBool; use std::sync::Mutex; -// use crate::passthrough::XrPassthroughLayer; +use crate::passthrough::{Passthrough, PassthroughLayer}; use crate::resource_macros::*; +use crate::xr::sys::CompositionLayerPassthroughFB; +use crate::xr::{CompositionLayerBase, CompositionLayerFlags}; use bevy::prelude::*; +use core::ptr; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); @@ -16,6 +19,8 @@ xr_arc_resource_wrapper!(XrFrameWaiter, Mutex); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); xr_arc_resource_wrapper!(XrFrameState, Mutex); xr_arc_resource_wrapper!(XrViews, Mutex>); +xr_arc_resource_wrapper!(XrPassthrough, Mutex); +xr_arc_resource_wrapper!(XrPassthroughLayer, Mutex); pub enum Swapchain { Vulkan(SwapchainInner), @@ -59,7 +64,7 @@ impl Swapchain { stage: &xr::Space, resolution: UVec2, environment_blend_mode: xr::EnvironmentBlendMode, - // passthrough_layer: Option<&XrPassthroughLayer>, + passthrough_layer: Option, ) -> xr::Result<()> { match self { Swapchain::Vulkan(swapchain) => swapchain.end( @@ -68,7 +73,7 @@ impl Swapchain { stage, resolution, environment_blend_mode, - // passthrough_layer, + passthrough_layer, ), } } @@ -128,7 +133,7 @@ impl SwapchainInner { stage: &xr::Space, resolution: UVec2, environment_blend_mode: xr::EnvironmentBlendMode, - // passthrough_layer: Option<&XrPassthroughLayer>, + passthrough_layer: Option, ) -> xr::Result<()> { let rect = xr::Rect2Di { offset: xr::Offset2Di { x: 0, y: 0 }, @@ -142,75 +147,79 @@ impl SwapchainInner { warn!("views are len of 0"); return Ok(()); } - // match passthrough_layer { - // Some(pass) => { - // // info!("Rendering with pass through"); - // let passthrough_layer = xr::sys::CompositionLayerPassthroughFB { - // ty: CompositionLayerPassthroughFB::TYPE, - // next: ptr::null(), - // flags: CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA, - // space: xr::sys::Space::NULL, - // layer_handle: pass.0, - // }; - // self.stream.lock().unwrap().end( - // predicted_display_time, - // environment_blend_mode, - // &[ - // &xr::CompositionLayerProjection::new() - // .layer_flags(CompositionLayerFlags::UNPREMULTIPLIED_ALPHA) - // .space(stage) - // .views(&[ - // xr::CompositionLayerProjectionView::new() - // .pose(views[0].pose) - // .fov(views[0].fov) - // .sub_image( - // xr::SwapchainSubImage::new() - // .swapchain(&swapchain) - // .image_array_index(0) - // .image_rect(rect), - // ), - // xr::CompositionLayerProjectionView::new() - // .pose(views[1].pose) - // .fov(views[1].fov) - // .sub_image( - // xr::SwapchainSubImage::new() - // .swapchain(&swapchain) - // .image_array_index(1) - // .image_rect(rect), - // ), - // ]), - // unsafe { - // &*(&passthrough_layer as *const _ as *const CompositionLayerBase) - // }, - // ], - // ) - // } + match passthrough_layer { + Some(pass) => { + //bevy::log::info!("Rendering with pass through"); - // None => - self.stream.lock().unwrap().end( - predicted_display_time, - environment_blend_mode, - &[&xr::CompositionLayerProjection::new().space(stage).views(&[ - xr::CompositionLayerProjectionView::new() - .pose(views[0].pose) - .fov(views[0].fov) - .sub_image( - xr::SwapchainSubImage::new() - .swapchain(&swapchain) - .image_array_index(0) - .image_rect(rect), - ), - xr::CompositionLayerProjectionView::new() - .pose(views[1].pose) - .fov(views[1].fov) - .sub_image( - xr::SwapchainSubImage::new() - .swapchain(&swapchain) - .image_array_index(1) - .image_rect(rect), - ), - ])], - ) - // } + let passthrough_layer = xr::sys::CompositionLayerPassthroughFB { + ty: CompositionLayerPassthroughFB::TYPE, + next: ptr::null(), + flags: CompositionLayerFlags::UNPREMULTIPLIED_ALPHA, + space: xr::sys::Space::NULL, + layer_handle: pass.0, + }; + + self.stream.lock().unwrap().end( + predicted_display_time, + environment_blend_mode, + &[ + unsafe { + &*(&passthrough_layer as *const _ as *const CompositionLayerBase) + }, + &xr::CompositionLayerProjection::new() + .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA) + .space(stage) + .views(&[ + xr::CompositionLayerProjectionView::new() + .pose(views[0].pose) + .fov(views[0].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(0) + .image_rect(rect), + ), + xr::CompositionLayerProjectionView::new() + .pose(views[1].pose) + .fov(views[1].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(1) + .image_rect(rect), + ), + ]), + ], + ) + } + + None => { + bevy::log::info!("Rendering without pass through"); + self.stream.lock().unwrap().end( + predicted_display_time, + environment_blend_mode, + &[&xr::CompositionLayerProjection::new().space(stage).views(&[ + xr::CompositionLayerProjectionView::new() + .pose(views[0].pose) + .fov(views[0].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(0) + .image_rect(rect), + ), + xr::CompositionLayerProjectionView::new() + .pose(views[1].pose) + .fov(views[1].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(1) + .image_rect(rect), + ), + ])], + ) + } + } } } diff --git a/src/xr_init.rs b/src/xr_init.rs index 499e7ce..88315c0 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -17,9 +17,10 @@ use wgpu::Instance; use crate::{ input::XrInput, + passthrough::{Passthrough, PassthroughLayer}, resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrPassthrough, + XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }, }; @@ -45,6 +46,9 @@ pub struct XrRenderData { pub xr_input: XrInput, pub xr_views: XrViews, pub xr_frame_state: XrFrameState, + pub xr_passthrough_active: bool, + pub xr_passthrough: XrPassthrough, + pub xr_passthrough_layer: XrPassthroughLayer, } #[derive(Event, Clone, Copy, Debug)] From 9704607c8c9ed19ac8d6c9f9c5372fbbe68c8100 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 5 Feb 2024 02:27:27 +0100 Subject: [PATCH 25/80] stuff --- src/graphics/vulkan.rs | 4 +++- src/lib.rs | 43 +++++++++++++++++++++++---------------- src/resource_macros.rs | 29 ++++++++++++++++++++++++++ src/resources.rs | 18 +++++++++++++--- src/xr_input/mod.rs | 3 ++- src/xr_input/xr_camera.rs | 27 ++++++++++++++++++++++-- 6 files changed, 100 insertions(+), 24 deletions(-) diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index ac0948d..f4ca0f0 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -356,7 +356,7 @@ pub fn start_xr_session( }); let swapchain_format = surface .as_ref() - .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0]) + .map(|surface| surface.get_capabilities(wgpu_adapter).formats[0]) .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb); let resolution = uvec2( @@ -387,6 +387,7 @@ pub fn start_xr_session( let buffers = images .into_iter() .map(|color_image| { + info!("image map swapchain"); let color_image = vk::Image::from_raw(color_image); let wgpu_hal_texture = unsafe { ::Device::texture_from_raw( @@ -449,6 +450,7 @@ pub fn start_xr_session( .into(), XrInput::new(xr_instance, &session.into_any_graphics())?, Vec::default().into(), + // Feels wrong to return a FrameState here, we probably should just wait for the next frame xr::FrameState { predicted_display_time: xr::Time::from_nanos(1), predicted_display_period: xr::Duration::from_nanos(1), diff --git a/src/lib.rs b/src/lib.rs index 80382aa..83b09f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -154,20 +154,20 @@ impl Plugin for OpenXrPlugin { .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), ); - render_app.add_systems( - Render, - ( - locate_views, - xr_input::xr_camera::xr_camera_head_sync, - sync_simple_transforms, - propagate_transforms, - update_cam_views, - ) - .chain() - .run_if(xr_only()) - // .run_if(xr_render_only()) - .in_set(RenderSet::Prepare), - ); + // render_app.add_systems( + // Render, + // ( + // locate_views, + // xr_input::xr_camera::xr_camera_head_sync_render, + // // sync_simple_transforms, + // // propagate_transforms, + // // update_cam_views, + // ) + // .chain() + // .run_if(xr_only()) + // // .run_if(xr_render_only()) + // .in_set(RenderSet::Prepare), + // ); render_app.add_systems( Render, xr_end_frame @@ -187,6 +187,8 @@ impl Plugin for OpenXrPlugin { } } +// Confirmed Working +// Not Working Actually, the cam doesn't render with the new pose for some reason fn update_cam_views(mut query: Query<(&mut ExtractedView, &GlobalTransform)>) { for (mut view, transform) in &mut query { view.transform = *transform; @@ -199,7 +201,6 @@ fn xr_skip_frame( environment_blend_mode: Res, ) { let swapchain: &Swapchain = &xr_swapchain; - // swapchain.begin().unwrap(); match swapchain { Swapchain::Vulkan(swap) => { swap.stream @@ -344,10 +345,14 @@ pub fn xr_wait_frame( return; } }; - frame_state.predicted_display_time = xr::Time::from_nanos( + info!( + "Post Wait Time: {}", frame_state.predicted_display_time.as_nanos() - + frame_state.predicted_display_period.as_nanos(), ); + // frame_state.predicted_display_time = xr::Time::from_nanos( + // frame_state.predicted_display_time.as_nanos() + // + frame_state.predicted_display_period.as_nanos(), + // ); info!("Post Frame Wait"); **should_render = frame_state.should_render; **waited = true; @@ -406,6 +411,10 @@ pub fn xr_end_frame( } { let _span = info_span!("xr_end_frame").entered(); + info!( + "End Frame Time: {}", + xr_frame_state.predicted_display_time.as_nanos() + ); let result = swapchain.end( xr_frame_state.predicted_display_time, &views, diff --git a/src/resource_macros.rs b/src/resource_macros.rs index 735ec50..bb83a63 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -32,6 +32,35 @@ macro_rules! xr_resource_wrapper { }; } +#[macro_export] +macro_rules! xr_resource_wrapper_no_extract { + ($wrapper_type:ident, $xr_type:ty) => { + #[derive( + Clone, Copy, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut, + )] + pub struct $wrapper_type($xr_type); + + impl $wrapper_type { + pub fn new(value: $xr_type) -> Self { + Self(value) + } + } + + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // &self.0 + // } + // } + + impl From<$xr_type> for $wrapper_type { + fn from(value: $xr_type) -> Self { + Self::new(value) + } + } + }; +} #[macro_export] macro_rules! xr_arc_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { diff --git a/src/resources.rs b/src/resources.rs index da72cf6..2854abc 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -4,10 +4,10 @@ use std::sync::Mutex; use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; -use crate::resource_macros::*; use crate::xr_init::XrStatus; +use crate::{resource_macros::*, xr_resource_wrapper_no_extract}; use bevy::prelude::*; -use bevy::render::extract_resource::ExtractResourcePlugin; +use bevy::render::extract_resource::{ExtractResource, ExtractResourcePlugin}; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); @@ -15,12 +15,24 @@ xr_resource_wrapper!(XrSession, xr::Session); xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); xr_resource_wrapper!(XrResolution, UVec2); xr_resource_wrapper!(XrFormat, wgpu::TextureFormat); -xr_resource_wrapper!(XrFrameState, xr::FrameState); +xr_resource_wrapper_no_extract!(XrFrameState, xr::FrameState); xr_resource_wrapper!(XrViews, Vec); xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); xr_no_clone_resource_wrapper!(XrFrameWaiter, xr::FrameWaiter); +impl ExtractResource for XrFrameState { + type Source = Self; + + fn extract_resource(source: &Self::Source) -> Self { + let mut state = *source; + state.predicted_display_time = xr::Time::from_nanos( + state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(), + ); + state + } +} + pub(crate) struct VulkanOXrSessionSetupInfo { pub(crate) device_ptr: *const c_void, pub(crate) physical_device_ptr: *const c_void, diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index ac2e13b..ee88e4d 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -36,7 +36,7 @@ use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, }; -use self::xr_camera::{XrCameraType, TransformExtract}; +use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCameraType}; #[derive(Copy, Clone)] pub struct OpenXrInput { @@ -87,6 +87,7 @@ impl Plugin for OpenXrInput { app.add_plugins(ExtractComponentPlugin::::default()); app.add_plugins(ExtractComponentPlugin::::default()); app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); } } diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 912b3ec..50d38ed 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -7,7 +7,7 @@ use bevy::prelude::*; use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; use bevy::render::extract_component::ExtractComponent; use bevy::render::primitives::Frustum; -use bevy::render::view::{ColorGrading, VisibleEntities}; +use bevy::render::view::{ColorGrading, ExtractedView, VisibleEntities}; use openxr::Fovf; #[derive(Bundle)] @@ -79,6 +79,7 @@ impl ExtractComponent for GlobalTransformExtract { Some(*item) } } + #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, @@ -271,7 +272,7 @@ impl CameraProjection for XRProjection { } pub fn xr_camera_head_sync( - views: ResMut, + views: Res, mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, ) { //TODO calculate HMD position @@ -290,3 +291,25 @@ pub fn xr_camera_head_sync( transform.translation = view.pose.position.to_vec3(); } } +pub fn xr_camera_head_sync_render( + views: Res, + mut query: Query<(&mut ExtractedView, &XrCameraType)>, +) { + //TODO calculate HMD position + for (mut transform, camera_type) in query.iter_mut() { + // let mut t = Transform::IDENTITY; + // let view_idx = match camera_type { + // XrCameraType::Xr(eye) => *eye as usize, + // // I don't belive we need a flatscrenn cam, that's just a cam without this component + // XrCameraType::Flatscreen => continue, + // }; + // let view = match views.get(view_idx) { + // Some(views) => views, + // None => continue, + // }; + // t.rotation = view.pose.orientation.to_quat(); + // t.translation = view.pose.position.to_vec3(); + info!("cam update"); + transform.transform = GlobalTransform::IDENTITY; + } +} From 74c0813c487fcc9187fd0efe643470376be8ff0f Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 5 Feb 2024 02:43:03 +0100 Subject: [PATCH 26/80] wip --- src/lib.rs | 24 +----------------------- src/resource_macros.rs | 9 +++++++-- src/resources.rs | 28 ++++++++-------------------- 3 files changed, 16 insertions(+), 45 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 83b09f1..647b666 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -154,20 +154,6 @@ impl Plugin for OpenXrPlugin { .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), ); - // render_app.add_systems( - // Render, - // ( - // locate_views, - // xr_input::xr_camera::xr_camera_head_sync_render, - // // sync_simple_transforms, - // // propagate_transforms, - // // update_cam_views, - // ) - // .chain() - // .run_if(xr_only()) - // // .run_if(xr_render_only()) - // .in_set(RenderSet::Prepare), - // ); render_app.add_systems( Render, xr_end_frame @@ -187,14 +173,6 @@ impl Plugin for OpenXrPlugin { } } -// Confirmed Working -// Not Working Actually, the cam doesn't render with the new pose for some reason -fn update_cam_views(mut query: Query<(&mut ExtractedView, &GlobalTransform)>) { - for (mut view, transform) in &mut query { - view.transform = *transform; - } -} - fn xr_skip_frame( xr_swapchain: Res, xr_frame_state: Res, @@ -238,7 +216,7 @@ impl PluginGroup for DefaultXrPlugins { ..default() }, }) - // .disable::() + .disable::() .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, diff --git a/src/resource_macros.rs b/src/resource_macros.rs index bb83a63..11ffd20 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -33,10 +33,15 @@ macro_rules! xr_resource_wrapper { } #[macro_export] -macro_rules! xr_resource_wrapper_no_extract { +macro_rules! xr_resource_wrapper_copy { ($wrapper_type:ident, $xr_type:ty) => { #[derive( - Clone, Copy, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut, + Clone, + Copy, + bevy::prelude::Resource, + bevy::prelude::Deref, + bevy::prelude::DerefMut, + bevy::render::extract_resource::ExtractResource, )] pub struct $wrapper_type($xr_type); diff --git a/src/resources.rs b/src/resources.rs index 2854abc..41678bb 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -4,36 +4,24 @@ use std::sync::Mutex; use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; -use crate::xr_init::XrStatus; -use crate::{resource_macros::*, xr_resource_wrapper_no_extract}; +use crate::{resource_macros::*, xr_resource_wrapper_copy}; use bevy::prelude::*; -use bevy::render::extract_resource::{ExtractResource, ExtractResourcePlugin}; +use bevy::render::extract_resource::ExtractResourcePlugin; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); xr_resource_wrapper!(XrSession, xr::Session); -xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); -xr_resource_wrapper!(XrResolution, UVec2); -xr_resource_wrapper!(XrFormat, wgpu::TextureFormat); -xr_resource_wrapper_no_extract!(XrFrameState, xr::FrameState); +xr_resource_wrapper_copy!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); +xr_resource_wrapper_copy!(XrResolution, UVec2); +xr_resource_wrapper_copy!(XrFormat, wgpu::TextureFormat); +xr_resource_wrapper_copy!(XrFrameState, xr::FrameState); xr_resource_wrapper!(XrViews, Vec); xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); xr_no_clone_resource_wrapper!(XrFrameWaiter, xr::FrameWaiter); -impl ExtractResource for XrFrameState { - type Source = Self; - fn extract_resource(source: &Self::Source) -> Self { - let mut state = *source; - state.predicted_display_time = xr::Time::from_nanos( - state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(), - ); - state - } -} - -pub(crate) struct VulkanOXrSessionSetupInfo { +pub struct VulkanOXrSessionSetupInfo { pub(crate) device_ptr: *const c_void, pub(crate) physical_device_ptr: *const c_void, pub(crate) vk_instance_ptr: *const c_void, @@ -41,7 +29,7 @@ pub(crate) struct VulkanOXrSessionSetupInfo { pub(crate) xr_system_id: xr::SystemId, } -pub(crate) enum OXrSessionSetupInfo { +pub enum OXrSessionSetupInfo { Vulkan(VulkanOXrSessionSetupInfo), } From 3803968b7c7725a88fed4abb99dbc8cda910666e Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 12 Feb 2024 09:05:44 +0100 Subject: [PATCH 27/80] clean up passthrough, change hand entity behavior and add cleanup steps to some plugins --- examples/android/Cargo.toml | 8 +- examples/android/src/lib.rs | 36 +++++--- examples/demo/src/lib.rs | 4 +- examples/globe.rs | 3 +- examples/xr.rs | 3 +- src/lib.rs | 23 +++-- src/passthrough.rs | 61 ++++++------- src/xr_init/mod.rs | 3 +- src/xr_input/actions.rs | 18 +++- src/xr_input/hands/common.rs | 133 +++++++++++----------------- src/xr_input/hands/emulated.rs | 10 +-- src/xr_input/hands/hand_tracking.rs | 9 +- src/xr_input/hands/mod.rs | 62 +++++++------ 13 files changed, 178 insertions(+), 195 deletions(-) diff --git a/examples/android/Cargo.toml b/examples/android/Cargo.toml index bea6cb3..1c412bc 100644 --- a/examples/android/Cargo.toml +++ b/examples/android/Cargo.toml @@ -16,10 +16,10 @@ bevy_oxr.path = "../.." bevy = "0.12" openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = ["mint"] } -[profile.release] -lto = "fat" -codegen-units = 1 -panic = "abort" +# [profile.release] +# lto = "fat" +# codegen-units = 1 +# panic = "abort" # This metadata is used by `cargo-apk` - `xbuild` uses the `manifest.yaml` instead. [package.metadata.android] diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index efb835a..ba0f411 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -3,10 +3,10 @@ use bevy::prelude::*; use bevy::transform::components::Transform; use bevy_oxr::graphics::extensions::XrExtensions; use bevy_oxr::graphics::XrAppInfo; -use bevy_oxr::graphics::XrPreferdBlendMode::AlphaBlend; -use bevy_oxr::passthrough::{passthrough_layer_pause, passthrough_layer_resume}; -use bevy_oxr::xr_init::XrRenderData; +use bevy_oxr::passthrough::{PausePassthrough, ResumePassthrough, XrPassthroughState}; +use bevy_oxr::xr_init::xr_only; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; +use bevy_oxr::xr_input::hands::HandBone; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker, @@ -25,16 +25,23 @@ fn main() { }, prefered_blend_mode: bevy_oxr::graphics::XrPreferdBlendMode::Opaque, }) - .add_plugins(OpenXrDebugRenderer) + // .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) .add_systems(Startup, setup) - .add_systems(Update, (proto_locomotion, toggle_passthrough)) + .add_systems(Update, (proto_locomotion, toggle_passthrough).run_if(xr_only())) + .add_systems(Update, debug_hand_render.run_if(xr_only())) .add_systems(Startup, spawn_controllers_example) .insert_resource(PrototypeLocomotionConfig::default()) .run(); } +fn debug_hand_render(query: Query<&GlobalTransform, With>, mut gizmos: Gizmos) { + for transform in &query { + gizmos.sphere(transform.translation(), Quat::IDENTITY, 0.01, Color::RED); + } +} + /// set up a simple 3D scene fn setup( mut commands: Commands, @@ -90,15 +97,18 @@ fn spawn_controllers_example(mut commands: Commands) { )); } -// Does this work? Not getting logs -fn toggle_passthrough(keys: Res>, mut xr_data: ResMut) { +// TODO: make this a vr button +fn toggle_passthrough( + keys: Res>, + passthrough_state: Res, + mut resume: EventWriter, + mut pause: EventWriter, +) { if keys.just_pressed(KeyCode::Space) { - if xr_data.xr_passthrough_active { - passthrough_layer_pause(xr_data); - bevy::log::info!("Passthrough paused"); - } else { - passthrough_layer_resume(xr_data); - bevy::log::info!("Passthrough resumed"); + match *passthrough_state { + XrPassthroughState::Unsupported => {} + XrPassthroughState::Running => pause.send_default(), + XrPassthroughState::Paused => resume.send_default(), } } } diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index d55db1a..8f369f9 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -23,7 +23,7 @@ use bevy_oxr::{ xr_input::{ actions::XrActionSets, debug_gizmos::OpenXrDebugRenderer, - hands::common::{HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput}, + hands::common::{HandInputDebugRenderer, HandResource, HandsResource}, hands::HandBone, interactions::{ draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions, @@ -120,7 +120,7 @@ pub fn main() { //test capsule .add_systems(Startup, spawn_capsule) //physics hands - .add_plugins(OpenXrHandInput) + // .add_plugins(OpenXrHandInput) .add_plugins(HandInputDebugRenderer) .add_systems(Startup, spawn_physics_hands) .add_systems( diff --git a/examples/globe.rs b/examples/globe.rs index 192916b..3f439be 100644 --- a/examples/globe.rs +++ b/examples/globe.rs @@ -3,7 +3,7 @@ use bevy::prelude::*; use bevy::transform::components::Transform; use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::resources::XrViews; -use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput}; +use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer; use bevy_oxr::xr_input::interactions::{ InteractionEvent, XRDirectInteractor, XRInteractorState, XRRayInteractor, XRSocketInteractor, }; @@ -31,7 +31,6 @@ fn main() { .add_systems(Update, (proto_locomotion, pull_to_ground).chain()) .insert_resource(PrototypeLocomotionConfig::default()) .add_systems(Startup, spawn_controllers_example) - .add_plugins(OpenXrHandInput) .add_plugins(HandInputDebugRenderer) .add_event::() .run(); diff --git a/examples/xr.rs b/examples/xr.rs index 6f8d2de..68104d9 100644 --- a/examples/xr.rs +++ b/examples/xr.rs @@ -7,7 +7,7 @@ use bevy_oxr::input::XrInput; use bevy_oxr::resources::{XrFrameState, XrSession}; use bevy_oxr::xr_input::actions::XrActionSets; -use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput}; +use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer; use bevy_oxr::xr_input::interactions::{ draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions, update_interactable_states, InteractionEvent, Touched, XRDirectInteractor, XRInteractable, @@ -39,7 +39,6 @@ fn main() { .add_systems(Update, proto_locomotion) .insert_resource(PrototypeLocomotionConfig::default()) .add_systems(Startup, spawn_controllers_example) - .add_plugins(OpenXrHandInput) .add_plugins(HandInputDebugRenderer) .add_systems( Update, diff --git a/src/lib.rs b/src/lib.rs index ae0a5a4..17c5e6c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,11 +31,13 @@ use openxr as xr; use passthrough::{PassthroughPlugin, XrPassthroughLayer, XrPassthroughState}; use resources::*; use xr_init::{ - xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrHasWaited, - XrShouldRender, XrStatus, + xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, SetupXrData, XrEarlyInitPlugin, + XrHasWaited, XrShouldRender, XrStatus, }; use xr_input::controllers::XrControllerType; -use xr_input::hands::XrHandPlugins; +use xr_input::hands::emulated::HandEmulationPlugin; +use xr_input::hands::hand_tracking::HandTrackingPlugin; +use xr_input::hands::HandPlugin; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -51,11 +53,6 @@ pub struct OpenXrPlugin { app_info: XrAppInfo, } -fn mr_test(mut commands: Commands, mut resume: EventWriter) { - commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); - resume.send_default(); -} - impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { app.insert_resource(XrSessionRunning::new(AtomicBool::new(false))); @@ -114,7 +111,6 @@ impl Plugin for OpenXrPlugin { app.add_plugins(RenderPlugin::default()); app.insert_resource(XrStatus::Disabled); } - app.add_systems(Update, mr_test.run_if(xr_only())); app.add_systems( PreUpdate, xr_poll_events.run_if(|status: Res| *status != XrStatus::NoInstance), @@ -146,8 +142,6 @@ impl Plugin for OpenXrPlugin { .run_if(xr_only()) .run_if(xr_after_wait_only()) .run_if(xr_render_only()) - // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet - // right before rendering .before(render_system) .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), @@ -224,7 +218,9 @@ impl PluginGroup for DefaultXrPlugins { .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) .add_after::(XrInitPlugin) .add_before::(XrEarlyInitPlugin) - .add(XrHandPlugins) + .add(HandPlugin) + .add(HandTrackingPlugin) + .add(HandEmulationPlugin) .add(PassthroughPlugin) .add(XrResourcePlugin) .set(WindowPlugin { @@ -259,6 +255,7 @@ fn xr_poll_events( session: Option>, session_running: Res, mut app_exit: EventWriter, + mut setup_xr: EventWriter, mut cleanup_xr: EventWriter, ) { if let (Some(instance), Some(session)) = (instance, session) { @@ -274,6 +271,7 @@ fn xr_poll_events( xr::SessionState::READY => { info!("Calling Session begin :3"); session.begin(VIEW_TYPE).unwrap(); + setup_xr.send_default(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); } xr::SessionState::STOPPING => { @@ -283,7 +281,6 @@ fn xr_poll_events( } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { // app_exit.send(AppExit); - return; } _ => {} diff --git a/src/passthrough.rs b/src/passthrough.rs index 51256e7..e6eb1e5 100644 --- a/src/passthrough.rs +++ b/src/passthrough.rs @@ -25,10 +25,11 @@ pub enum XrPassthroughState { Paused, } -pub struct PassthroughPlugin; xr_arc_resource_wrapper!(XrPassthrough, xr::Passthrough); xr_arc_resource_wrapper!(XrPassthroughLayer, xr::PassthroughLayer); +pub struct PassthroughPlugin; + impl Plugin for PassthroughPlugin { fn build(&self, app: &mut App) { app.add_event::(); @@ -78,20 +79,31 @@ fn check_passthrough_support(mut cmds: Commands, instance: Option, mut state: ResMut) { +fn resume_passthrough( + layer: Res, + mut state: ResMut, + mut clear_color: ResMut, +) { if let Err(e) = layer.resume() { warn!("Unable to resume Passthrough: {}", e); return; } - info!("<=> Resume Passthrough"); + clear_color.set_a(0.0); + clear_color.set_r(0.0); + clear_color.set_g(0.0); + clear_color.set_b(0.0); *state = XrPassthroughState::Running; } -fn pause_passthrough(layer: Res, mut state: ResMut) { +fn pause_passthrough( + layer: Res, + mut state: ResMut, + mut clear_color: ResMut, +) { if let Err(e) = layer.pause() { warn!("Unable to resume Passthrough: {}", e); return; } - info!("<=> Pausing Passthrough"); + clear_color.set_a(1.0); *state = XrPassthroughState::Paused; } @@ -100,8 +112,8 @@ fn cleanup_passthrough(mut cmds: Commands) { cmds.remove_resource::(); } -fn setup_passthrough(mut cmds: Commands, instance: Res, session: Res) { - match create_passthrough(&instance, &session) { +fn setup_passthrough(mut cmds: Commands, session: Res) { + match create_passthrough(&session) { Ok((passthrough, layer)) => { cmds.insert_resource(XrPassthrough::from(passthrough)); cmds.insert_resource(XrPassthroughLayer::from(layer)); @@ -181,7 +193,6 @@ pub fn supports_passthrough(instance: &XrInstance, system: xr::SystemId) -> xr:: #[inline] pub fn create_passthrough( - instance: &XrInstance, xr_session: &XrSession, ) -> xr::Result<(xr::Passthrough, xr::PassthroughLayer)> { let passthrough = match xr_session { @@ -199,28 +210,14 @@ pub fn create_passthrough( Ok((passthrough, passthrough_layer)) } -// #[inline] -// pub fn passthrough_layer_resume(mut layer: ResMut, mut passthrough: ResMut) -> xr::Result<()> { -// layer.resume() -// } +/// Enable Passthrough on xr startup +/// just sends the [`ResumePassthrough`] event in [`XrSetup`] +pub struct EnablePassthroughStartup; -// #[inline] -// pub fn passthrough_layer_pause(mut xr_data_resource: ResMut) -> xr::Result<()> { -// unsafe { -// let passthrough_layer = &xr_data_resource.xr_passthrough_layer; -// { -// let passthrough_layer_locked = passthrough_layer.lock().unwrap(); -// cvt((xr_data_resource -// .xr_instance -// .exts() -// .fb_passthrough -// .unwrap() -// .passthrough_layer_pause)( -// *passthrough_layer_locked -// ))?; -// } -// xr_data_resource.xr_passthrough_active = false; -// bevy::log::info!("Paused passthrough layer"); -// Ok(()) -// } -// } +impl Plugin for EnablePassthroughStartup { + fn build(&self, app: &mut App) { + app.add_systems(XrSetup, |mut e: EventWriter| { + e.send_default(); + }); + } +} diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs index 06d9578..bd17a04 100644 --- a/src/xr_init/mod.rs +++ b/src/xr_init/mod.rs @@ -126,7 +126,7 @@ pub struct StartXrSession; pub struct EndXrSession; #[derive(Event, Clone, Copy, Default)] -struct SetupXrData; +pub(crate) struct SetupXrData; #[derive(Event, Clone, Copy, Default)] pub(crate) struct CleanupXrData; @@ -192,7 +192,6 @@ fn start_xr_session( commands.insert_resource(xr_views); commands.insert_resource(xr_frame_state); *status = XrStatus::Enabling; - setup_xr.send_default(); } fn stop_xr_session(session: ResMut, mut status: ResMut) { diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 748968d..b9d8c2d 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -6,7 +6,7 @@ use xr::{Action, Binding, Haptic, Posef, Vector2f}; use crate::{ resources::{XrInstance, XrSession}, - xr_init::XrPrePostSetup, + xr_init::{XrCleanup, XrPrePostSetup, XrPreSetup}, }; use super::oculus_touch::ActionSets; @@ -16,13 +16,23 @@ pub use xr::sys::NULL_PATH; pub struct OpenXrActionsPlugin; impl Plugin for OpenXrActionsPlugin { fn build(&self, app: &mut App) { - app.insert_resource(SetupActionSets { - sets: HashMap::new(), - }); + app.add_systems(XrPreSetup, insert_setup_action_sets); app.add_systems(XrPrePostSetup, setup_oxr_actions); + app.add_systems(XrCleanup, clean_actions); } } +fn insert_setup_action_sets(mut cmds: Commands) { + cmds.insert_resource(SetupActionSets { + sets: HashMap::new(), + }); +} + +fn clean_actions(mut cmds: Commands) { + cmds.remove_resource::(); + cmds.remove_resource::(); +} + #[inline(always)] fn create_action( action: &SetupAction, diff --git a/src/xr_input/hands/common.rs b/src/xr_input/hands/common.rs index 104c6b3..f080613 100644 --- a/src/xr_input/hands/common.rs +++ b/src/xr_input/hands/common.rs @@ -1,6 +1,10 @@ -use bevy::prelude::{ - default, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, Plugin, PostUpdate, - Query, Resource, SpatialBundle, Startup, Transform, +use bevy::{ + core::Name, + prelude::{ + default, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, Plugin, PostUpdate, + Query, Resource, SpatialBundle, Startup, Transform, + }, + transform::components::GlobalTransform, }; use crate::xr_input::{trackers::OpenXRTracker, Hand}; @@ -8,14 +12,14 @@ use crate::xr_input::{trackers::OpenXRTracker, Hand}; use super::{BoneTrackingStatus, HandBone}; /// add debug renderer for controllers -#[derive(Default)] -pub struct OpenXrHandInput; - -impl Plugin for OpenXrHandInput { - fn build(&self, app: &mut bevy::prelude::App) { - app.add_systems(Startup, spawn_hand_entities); - } -} +// #[derive(Default)] +// pub struct OpenXrHandInput; +// +// impl Plugin for OpenXrHandInput { +// fn build(&self, app: &mut bevy::prelude::App) { +// app.add_systems(Startup, spawn_hand_entities); +// } +// } /// add debug renderer for controllers #[derive(Default)] @@ -161,75 +165,46 @@ pub fn spawn_hand_entities(mut commands: Commands) { for bone in bones.iter() { let boneid = commands .spawn(( + Name::new(format!("{:?} {:?}", hand, bone)), SpatialBundle::default(), - bone.clone(), + *bone, OpenXRTracker, - hand.clone(), + *hand, BoneTrackingStatus::Emulated, HandBoneRadius(0.1), )) .id(); - match hand { - Hand::Left => match bone { - HandBone::Palm => hand_resource.left.palm = boneid, - HandBone::Wrist => hand_resource.left.wrist = boneid, - HandBone::ThumbMetacarpal => hand_resource.left.thumb.metacarpal = boneid, - HandBone::ThumbProximal => hand_resource.left.thumb.proximal = boneid, - HandBone::ThumbDistal => hand_resource.left.thumb.distal = boneid, - HandBone::ThumbTip => hand_resource.left.thumb.tip = boneid, - HandBone::IndexMetacarpal => hand_resource.left.index.metacarpal = boneid, - HandBone::IndexProximal => hand_resource.left.index.proximal = boneid, - HandBone::IndexIntermediate => hand_resource.left.index.intermediate = boneid, - HandBone::IndexDistal => hand_resource.left.index.distal = boneid, - HandBone::IndexTip => hand_resource.left.index.tip = boneid, - HandBone::MiddleMetacarpal => hand_resource.left.middle.metacarpal = boneid, - HandBone::MiddleProximal => hand_resource.left.middle.proximal = boneid, - HandBone::MiddleIntermediate => hand_resource.left.middle.intermediate = boneid, - HandBone::MiddleDistal => hand_resource.left.middle.distal = boneid, - HandBone::MiddleTip => hand_resource.left.middle.tip = boneid, - HandBone::RingMetacarpal => hand_resource.left.ring.metacarpal = boneid, - HandBone::RingProximal => hand_resource.left.ring.proximal = boneid, - HandBone::RingIntermediate => hand_resource.left.ring.intermediate = boneid, - HandBone::RingDistal => hand_resource.left.ring.distal = boneid, - HandBone::RingTip => hand_resource.left.ring.tip = boneid, - HandBone::LittleMetacarpal => hand_resource.left.little.metacarpal = boneid, - HandBone::LittleProximal => hand_resource.left.little.proximal = boneid, - HandBone::LittleIntermediate => hand_resource.left.little.intermediate = boneid, - HandBone::LittleDistal => hand_resource.left.little.distal = boneid, - HandBone::LittleTip => hand_resource.left.little.tip = boneid, - }, - Hand::Right => match bone { - HandBone::Palm => hand_resource.right.palm = boneid, - HandBone::Wrist => hand_resource.right.wrist = boneid, - HandBone::ThumbMetacarpal => hand_resource.right.thumb.metacarpal = boneid, - HandBone::ThumbProximal => hand_resource.right.thumb.proximal = boneid, - HandBone::ThumbDistal => hand_resource.right.thumb.distal = boneid, - HandBone::ThumbTip => hand_resource.right.thumb.tip = boneid, - HandBone::IndexMetacarpal => hand_resource.right.index.metacarpal = boneid, - HandBone::IndexProximal => hand_resource.right.index.proximal = boneid, - HandBone::IndexIntermediate => hand_resource.right.index.intermediate = boneid, - HandBone::IndexDistal => hand_resource.right.index.distal = boneid, - HandBone::IndexTip => hand_resource.right.index.tip = boneid, - HandBone::MiddleMetacarpal => hand_resource.right.middle.metacarpal = boneid, - HandBone::MiddleProximal => hand_resource.right.middle.proximal = boneid, - HandBone::MiddleIntermediate => { - hand_resource.right.middle.intermediate = boneid - } - HandBone::MiddleDistal => hand_resource.right.middle.distal = boneid, - HandBone::MiddleTip => hand_resource.right.middle.tip = boneid, - HandBone::RingMetacarpal => hand_resource.right.ring.metacarpal = boneid, - HandBone::RingProximal => hand_resource.right.ring.proximal = boneid, - HandBone::RingIntermediate => hand_resource.right.ring.intermediate = boneid, - HandBone::RingDistal => hand_resource.right.ring.distal = boneid, - HandBone::RingTip => hand_resource.right.ring.tip = boneid, - HandBone::LittleMetacarpal => hand_resource.right.little.metacarpal = boneid, - HandBone::LittleProximal => hand_resource.right.little.proximal = boneid, - HandBone::LittleIntermediate => { - hand_resource.right.little.intermediate = boneid - } - HandBone::LittleDistal => hand_resource.right.little.distal = boneid, - HandBone::LittleTip => hand_resource.right.little.tip = boneid, - }, + let hand_res = match hand { + Hand::Left => &mut hand_resource.left, + Hand::Right => &mut hand_resource.right, + }; + match bone { + HandBone::Palm => hand_res.palm = boneid, + HandBone::Wrist => hand_res.wrist = boneid, + HandBone::ThumbMetacarpal => hand_res.thumb.metacarpal = boneid, + HandBone::ThumbProximal => hand_res.thumb.proximal = boneid, + HandBone::ThumbDistal => hand_res.thumb.distal = boneid, + HandBone::ThumbTip => hand_res.thumb.tip = boneid, + HandBone::IndexMetacarpal => hand_res.index.metacarpal = boneid, + HandBone::IndexProximal => hand_res.index.proximal = boneid, + HandBone::IndexIntermediate => hand_res.index.intermediate = boneid, + HandBone::IndexDistal => hand_res.index.distal = boneid, + HandBone::IndexTip => hand_res.index.tip = boneid, + HandBone::MiddleMetacarpal => hand_res.middle.metacarpal = boneid, + HandBone::MiddleProximal => hand_res.middle.proximal = boneid, + HandBone::MiddleIntermediate => hand_res.middle.intermediate = boneid, + HandBone::MiddleDistal => hand_res.middle.distal = boneid, + HandBone::MiddleTip => hand_res.middle.tip = boneid, + HandBone::RingMetacarpal => hand_res.ring.metacarpal = boneid, + HandBone::RingProximal => hand_res.ring.proximal = boneid, + HandBone::RingIntermediate => hand_res.ring.intermediate = boneid, + HandBone::RingDistal => hand_res.ring.distal = boneid, + HandBone::RingTip => hand_res.ring.tip = boneid, + HandBone::LittleMetacarpal => hand_res.little.metacarpal = boneid, + HandBone::LittleProximal => hand_res.little.proximal = boneid, + HandBone::LittleIntermediate => hand_res.little.intermediate = boneid, + HandBone::LittleDistal => hand_res.little.distal = boneid, + HandBone::LittleTip => hand_res.little.tip = boneid, } } } @@ -241,16 +216,12 @@ pub struct HandBoneRadius(pub f32); pub fn draw_hand_entities( mut gizmos: Gizmos, - query: Query<(&Transform, &HandBone, &HandBoneRadius)>, + query: Query<(&GlobalTransform, &HandBone, &HandBoneRadius)>, ) { for (transform, hand_bone, hand_bone_radius) in query.iter() { let (_, color) = get_bone_gizmo_style(hand_bone); - gizmos.sphere( - transform.translation, - transform.rotation, - hand_bone_radius.0, - color, - ); + let (_, rotation, translation) = transform.to_scale_rotation_translation(); + gizmos.sphere(translation, rotation, hand_bone_radius.0, color); } } diff --git a/src/xr_input/hands/emulated.rs b/src/xr_input/hands/emulated.rs index ab1e627..9ebbc73 100644 --- a/src/xr_input/hands/emulated.rs +++ b/src/xr_input/hands/emulated.rs @@ -92,8 +92,6 @@ fn setup_hand_emulation_action_set(mut action_sets: ResMut) { suggest_oculus_touch_profile(action_set); } -pub struct EmulatedHandPoseData {} - fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) { action_set.suggest_binding( "/interaction_profiles/oculus/touch_controller", @@ -131,7 +129,6 @@ pub(crate) fn update_hand_skeleton_from_emulated( action_sets: Res, left_controller_transform: Query<&Transform, With>, right_controller_transform: Query<&Transform, With>, - tracking_root_transform: Query<&Transform, With>, mut bones: Query< ( &mut Transform, @@ -226,7 +223,6 @@ pub(crate) fn update_hand_skeleton_from_emulated( }, } } - let trt = tracking_root_transform.single(); for (mut t, bone, hand, status, mut radius) in bones.iter_mut() { match status { BoneTrackingStatus::Emulated => {} @@ -238,9 +234,9 @@ pub(crate) fn update_hand_skeleton_from_emulated( Hand::Left => 0, Hand::Right => 1, }][bone.get_index_from_bone()]; - *t = t.with_scale(trt.scale); - *t = t.with_rotation(trt.rotation * t.rotation); - *t = t.with_translation(trt.transform_point(t.translation)); + // *t = t.with_scale(trt.scale); + // *t = t.with_rotation(trt.rotation * t.rotation); + // *t = t.with_translation(trt.transform_point(t.translation)); } } pub fn update_hand_bones_emulated( diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index fd8c920..e2326a1 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -6,7 +6,7 @@ use crate::{ input::XrInput, resources::{XrFrameState, XrSession}, xr_init::xr_only, - xr_input::{hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv, Vec3Conv}, + xr_input::{hands::HandBone, Hand, QuatConv, Vec3Conv}, }; use super::BoneTrackingStatus; @@ -158,7 +158,6 @@ pub fn update_hand_bones( hand_tracking: Option>, xr_input: Res, xr_frame_state: Res, - root_query: Query<(&Transform, With, Without)>, mut bones: Query<( &mut Transform, &Hand, @@ -174,7 +173,6 @@ pub fn update_hand_bones( return; } }; - let (root_transform, _, _) = root_query.get_single().unwrap(); let left_hand_data = hand_ref.get_poses(Hand::Left); let right_hand_data = hand_ref.get_poses(Hand::Right); bones @@ -203,8 +201,7 @@ pub fn update_hand_bones( *status = BoneTrackingStatus::Tracked; } radius.0 = bone_data.radius; - *transform = transform - .with_translation(root_transform.transform_point(bone_data.position)) - .with_rotation(root_transform.rotation * bone_data.orientation) + transform.translation = bone_data.position; + transform.rotation = bone_data.orientation; }); } diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs index fa85cf8..8ab17d8 100644 --- a/src/xr_input/hands/mod.rs +++ b/src/xr_input/hands/mod.rs @@ -1,38 +1,50 @@ -use bevy::{app::PluginGroupBuilder, prelude::*}; +use bevy::prelude::*; use openxr::FormFactor; use crate::{ resources::{XrInstance, XrSession}, - xr_init::XrPreSetup, + xr_init::{XrCleanup, XrPreSetup, XrSetup}, }; use self::{ - emulated::HandEmulationPlugin, - hand_tracking::{DisableHandTracking, HandTrackingData, HandTrackingPlugin}, + common::{spawn_hand_entities, HandBoneRadius, HandsResource}, + hand_tracking::{DisableHandTracking, HandTrackingData}, }; +use super::{trackers::OpenXRTracker, Hand}; + pub mod common; pub mod emulated; pub mod hand_tracking; -pub struct XrHandPlugins; - -impl Plugin for XrHandPlugins { - fn build(&self, app: &mut App) { - app.add_plugins(HandTrackingPlugin) - .add_plugins(HandPlugin) - .add_plugins(HandEmulationPlugin); - } -} - pub struct HandPlugin; impl Plugin for HandPlugin { fn build(&self, app: &mut App) { app.add_systems(XrPreSetup, check_for_handtracking); + app.add_systems(XrSetup, spawn_hand_entities); + app.add_systems(XrCleanup, despawn_hand_entities); } } +#[allow(clippy::type_complexity)] +fn despawn_hand_entities( + mut commands: Commands, + hand_entities: Query< + Entity, + ( + With, + With, + With, + ), + >, +) { + for e in &hand_entities { + commands.entity(e).despawn_recursive(); + } + commands.remove_resource::() +} + fn check_for_handtracking( mut commands: Commands, instance: Res, @@ -86,21 +98,17 @@ pub enum HandBone { } impl HandBone { pub fn is_finger(&self) -> bool { - match &self { - HandBone::Wrist => false, - HandBone::Palm => false, - _ => true, - } + !matches!(self, HandBone::Wrist | HandBone::Palm) } pub fn is_metacarpal(&self) -> bool { - match &self { - HandBone::ThumbMetacarpal => true, - HandBone::IndexMetacarpal => true, - HandBone::MiddleMetacarpal => true, - HandBone::RingMetacarpal => true, - HandBone::LittleTip => true, - _ => false, - } + matches!( + self, + HandBone::ThumbMetacarpal + | HandBone::IndexMetacarpal + | HandBone::MiddleMetacarpal + | HandBone::RingMetacarpal + | HandBone::LittleTip + ) } pub const fn get_all_bones() -> [HandBone; 26] { [ From 31d7b05b4a288aea3e3abddfe476b14f9c02d85f Mon Sep 17 00:00:00 2001 From: Schmarni Date: Thu, 15 Feb 2024 06:04:16 +0100 Subject: [PATCH 28/80] handtracking not working on quest 2 v62 --- examples/android/src/lib.rs | 9 ++- src/lib.rs | 17 +++-- src/resources.rs | 1 - src/xr_init/mod.rs | 4 +- src/xr_input/actions.rs | 41 +++++++++-- src/xr_input/hands/common.rs | 2 +- src/xr_input/hands/hand_tracking.rs | 12 +++- src/xr_input/mod.rs | 93 ++++++++---------------- src/xr_input/xr_camera.rs | 105 ++++++++++++++++++---------- 9 files changed, 163 insertions(+), 121 deletions(-) diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index ba0f411..a52a93c 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -6,6 +6,7 @@ use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::passthrough::{PausePassthrough, ResumePassthrough, XrPassthroughState}; use bevy_oxr::xr_init::xr_only; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; +use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer; use bevy_oxr::xr_input::hands::HandBone; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ @@ -17,6 +18,7 @@ use bevy_oxr::DefaultXrPlugins; fn main() { let mut xr_extensions = XrExtensions::default(); xr_extensions.enable_fb_passthrough(); + xr_extensions.enable_hand_tracking(); App::new() .add_plugins(DefaultXrPlugins { reqeusted_extensions: xr_extensions, @@ -28,8 +30,13 @@ fn main() { // .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) + .add_plugins(HandInputDebugRenderer) + .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup) .add_systems(Startup, setup) - .add_systems(Update, (proto_locomotion, toggle_passthrough).run_if(xr_only())) + .add_systems( + Update, + (proto_locomotion, toggle_passthrough).run_if(xr_only()), + ) .add_systems(Update, debug_hand_render.run_if(xr_only())) .add_systems(Startup, spawn_controllers_example) .insert_resource(PrototypeLocomotionConfig::default()) diff --git a/src/lib.rs b/src/lib.rs index 17c5e6c..49c8508 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,12 +6,8 @@ pub mod resources; pub mod xr_init; pub mod xr_input; -use std::fs::File; -use std::io::{BufWriter, Write}; use std::sync::atomic::AtomicBool; -use std::sync::{Arc, Mutex}; -use crate::passthrough::ResumePassthrough; use crate::xr_init::{StartXrSession, XrInitPlugin}; use crate::xr_input::oculus_touch::ActionSets; use bevy::app::{AppExit, PluginGroupBuilder}; @@ -34,10 +30,12 @@ use xr_init::{ xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, SetupXrData, XrEarlyInitPlugin, XrHasWaited, XrShouldRender, XrStatus, }; +use xr_input::actions::OpenXrActionsPlugin; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::HandTrackingPlugin; use xr_input::hands::HandPlugin; +use xr_input::xr_camera::XrCameraPlugin; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -81,7 +79,7 @@ impl Plugin for OpenXrPlugin { debug!("Configured wgpu adapter Features: {:#?}", device.features()); warn!("Starting with OpenXR Instance"); app.insert_resource(xr_instance.clone()); - app.insert_resource(blend_mode.clone()); + app.insert_resource(blend_mode); app.insert_resource(ActionSets(vec![])); app.insert_resource(xr_instance); app.insert_resource(blend_mode); @@ -120,7 +118,9 @@ impl Plugin for OpenXrPlugin { ( xr_reset_per_frame_resources, xr_wait_frame.run_if(xr_only()), + // xr_begin_frame.run_if(xr_only()), locate_views.run_if(xr_only()), + apply_deferred, ) .chain() @@ -215,8 +215,10 @@ impl PluginGroup for DefaultXrPlugins { reqeusted_extensions: self.reqeusted_extensions, app_info: self.app_info.clone(), }) - .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) - .add_after::(XrInitPlugin) + .add(XrInitPlugin) + .add(OpenXrInput::new(XrControllerType::OculusTouch)) + .add(OpenXrActionsPlugin) + .add(XrCameraPlugin) .add_before::(XrEarlyInitPlugin) .add(HandPlugin) .add(HandTrackingPlugin) @@ -318,6 +320,7 @@ pub fn xr_wait_frame( return; } }; + // frame_state.predicted_display_time = xr::Time::from_nanos(frame_state.predicted_display_time.as_nanos() + frame_state.predicted_display_period.as_nanos()); **should_render = frame_state.should_render; **waited = true; } diff --git a/src/resources.rs b/src/resources.rs index 3b6ee26..305c76a 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -16,7 +16,6 @@ use core::ptr; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); -// xr_resource_wrapper!(XrSession, xr::Session); xr_resource_wrapper_copy!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); xr_resource_wrapper_copy!(XrResolution, UVec2); xr_resource_wrapper_copy!(XrFormat, wgpu::TextureFormat); diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs index bd17a04..70ac9c9 100644 --- a/src/xr_init/mod.rs +++ b/src/xr_init/mod.rs @@ -54,6 +54,7 @@ pub fn xr_after_wait_only() -> impl FnMut(Res) -> bool { impl Plugin for XrEarlyInitPlugin { fn build(&self, app: &mut App) { + add_schedules(app); app.add_event::() .add_event::() .add_event::() @@ -63,7 +64,6 @@ impl Plugin for XrEarlyInitPlugin { impl Plugin for XrInitPlugin { fn build(&self, app: &mut App) { - add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); @@ -106,7 +106,9 @@ fn setup_manual_texture_views( } pub fn setup_xr(world: &mut World) { + info!("Pre XrPreSetup"); world.run_schedule(XrPreSetup); + info!("Post XrPreSetup"); world.run_schedule(XrSetup); world.run_schedule(XrPrePostSetup); world.run_schedule(XrPostSetup); diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index b9d8c2d..940f40a 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -6,7 +6,7 @@ use xr::{Action, Binding, Haptic, Posef, Vector2f}; use crate::{ resources::{XrInstance, XrSession}, - xr_init::{XrCleanup, XrPrePostSetup, XrPreSetup}, + xr_init::{xr_only, XrCleanup, XrPrePostSetup, XrPreSetup}, }; use super::oculus_touch::ActionSets; @@ -16,13 +16,18 @@ pub use xr::sys::NULL_PATH; pub struct OpenXrActionsPlugin; impl Plugin for OpenXrActionsPlugin { fn build(&self, app: &mut App) { - app.add_systems(XrPreSetup, insert_setup_action_sets); + app.add_systems(PreUpdate, sync_actions.run_if(xr_only())); + app.add_systems( + XrPreSetup, + (insert_setup_action_sets, apply_deferred).chain(), + ); app.add_systems(XrPrePostSetup, setup_oxr_actions); app.add_systems(XrCleanup, clean_actions); } } fn insert_setup_action_sets(mut cmds: Commands) { + info!("WHAT?!"); cmds.insert_resource(SetupActionSets { sets: HashMap::new(), }); @@ -57,7 +62,6 @@ pub fn setup_oxr_actions(world: &mut World) { let right_path = instance.string_to_path("/user/hand/right").unwrap(); let hands = [left_path, right_path]; - let mut oxr_action_sets = Vec::new(); let mut action_sets = XrActionSets { sets: default() }; // let mut action_bindings: HashMap<&'static str, Vec> = HashMap::new(); let mut action_bindings: HashMap< @@ -101,11 +105,11 @@ pub fn setup_oxr_actions(world: &mut World) { } } } - oxr_action_sets.push(oxr_action_set); + // oxr_action_sets.push(oxr_action_set); action_sets.sets.insert( set_name, ActionSet { - // oxr_action_set, + oxr_action_set, actions, enabled: true, }, @@ -152,10 +156,15 @@ pub fn setup_oxr_actions(world: &mut World) { .expect("Unable to suggest interaction bindings!"); } session - .attach_action_sets(&oxr_action_sets.iter().collect::>()) + .attach_action_sets( + &action_sets + .sets + .values() + .map(|set| &set.oxr_action_set) + .collect::>(), + ) .expect("Unable to attach action sets!"); - world.insert_resource(ActionSets(oxr_action_sets)); world.insert_resource(action_sets); } @@ -267,6 +276,7 @@ pub struct ActionSet { // add functionality to enable/disable action sets enabled: bool, actions: HashMap<&'static str, TypedAction>, + oxr_action_set: xr::ActionSet, } #[derive(Resource)] @@ -380,3 +390,20 @@ impl XrActionSets { } } } + +pub fn sync_actions(action_sets: Res, session: Res) { + let active_sets = action_sets + .sets + .values() + .filter_map(|set| { + if set.enabled { + Some(xr::ActiveActionSet::new(&set.oxr_action_set)) + } else { + None + } + }) + .collect::>(); + if let Err(err) = session.sync_actions(&active_sets) { + warn!("OpenXR action sync error: {}", err); + } +} diff --git a/src/xr_input/hands/common.rs b/src/xr_input/hands/common.rs index f080613..7958e10 100644 --- a/src/xr_input/hands/common.rs +++ b/src/xr_input/hands/common.rs @@ -170,7 +170,7 @@ pub fn spawn_hand_entities(mut commands: Commands) { *bone, OpenXRTracker, *hand, - BoneTrackingStatus::Emulated, + BoneTrackingStatus::Tracked, HandBoneRadius(0.1), )) .id(); diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index e2326a1..0c632f8 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -63,6 +63,7 @@ pub struct HandJoint { pub radius: f32, } +#[derive(Debug)] pub struct HandJoints { inner: [HandJoint; 26], } @@ -175,9 +176,15 @@ pub fn update_hand_bones( }; let left_hand_data = hand_ref.get_poses(Hand::Left); let right_hand_data = hand_ref.get_poses(Hand::Right); + if left_hand_data.is_none() || right_hand_data.is_none() { + error!("something is very wrong for hand_tracking!! doesn't have data for both hands!"); + } + + info!("hand_tracking"); bones .par_iter_mut() .for_each(|(mut transform, hand, bone, mut radius, mut status)| { + info!("hand_tracking bone before filter"); match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) { (Hand::Left, Some(DisableHandTracking::OnlyLeft)) => { *status = BoneTrackingStatus::Emulated; @@ -189,14 +196,17 @@ pub fn update_hand_bones( } _ => {} } + info!("hand_tracking bone mid filter"); let bone_data = match (hand, &left_hand_data, &right_hand_data) { (Hand::Left, Some(data), _) => data.get_joint(*bone), (Hand::Right, _, Some(data)) => data.get_joint(*bone), - _ => { + (hand, left_data, right_data) => { + info!("{:?},{:?},{:?}", hand, left_data, right_data); *status = BoneTrackingStatus::Emulated; return; } }; + info!("hand_tracking bone after filter"); if *status == BoneTrackingStatus::Emulated { *status = BoneTrackingStatus::Tracked; } diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index ee88e4d..13c96df 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -10,7 +10,7 @@ pub mod trackers; pub mod xr_camera; use crate::resources::{XrInstance, XrSession}; -use crate::xr_init::{xr_only, XrPostSetup, XrPreSetup, XrSetup}; +use crate::xr_init::{xr_only, XrCleanup, XrPostSetup, XrPreSetup, XrSetup}; use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; @@ -19,6 +19,7 @@ use bevy::app::{App, PostUpdate, Startup}; use bevy::ecs::entity::Entity; use bevy::ecs::query::With; use bevy::ecs::system::Query; +use bevy::hierarchy::DespawnRecursiveExt; use bevy::log::{info, warn}; use bevy::math::Vec2; use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource}; @@ -36,7 +37,7 @@ use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, }; -use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCameraType}; +use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCamera}; #[derive(Copy, Clone)] pub struct OpenXrInput { @@ -56,9 +57,8 @@ impl OpenXrInput { impl Plugin for OpenXrInput { fn build(&self, app: &mut App) { - app.add_plugins(CameraProjectionPlugin::::default()); - app.add_plugins(OpenXrActionsPlugin); app.add_systems(XrPostSetup, post_action_setup_oculus_controller); + // why only when the controller is oculus? that is still backed by generic actions match self.controller_type { XrControllerType::OculusTouch => { app.add_systems(XrSetup, setup_oculus_controller); @@ -66,77 +66,42 @@ impl Plugin for OpenXrInput { } //adopt any new trackers app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only())); - app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); - app.add_systems( - PreUpdate, - xr_camera_head_sync - .run_if(xr_only()) - .after(xr_wait_frame) - .after(locate_views), - ); + // app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); //update controller trackers app.add_systems(Update, update_open_xr_controllers.run_if(xr_only())); - app.add_systems( - PostUpdate, - update_frusta:: - .after(TransformSystem::TransformPropagate) - .before(VisibilitySystems::UpdatePerspectiveFrusta), - ); app.add_systems(XrPreSetup, init_subaction_path); - app.add_systems(XrSetup, setup_xr_cameras); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); + app.add_systems(XrSetup, setup_xr_root); + app.add_systems(XrCleanup, cleanup_xr_root); } } -#[derive(Deref, DerefMut, Resource)] -pub struct InteractionProfileBindings(pub HashMap<&'static str, Vec>>); - -fn setup_binding_recommendations( - mut commands: Commands, - instance: Res, - bindings: Res, -) { - commands.remove_resource::(); -} - -fn setup_xr_cameras( +fn cleanup_xr_root( mut commands: Commands, tracking_root_query: Query>, ) { - //this needs to do the whole xr tracking volume not just cameras - //get the root? - - let tracking_root = match tracking_root_query.get_single() { - Ok(e) => e, - Err(_) => commands - .spawn((SpatialBundle::default(), OpenXRTrackingRoot)) - .id(), - }; - let right = commands - .spawn((XrCameraBundle::new(Eye::Right), OpenXRRightEye)) - .id(); - let left = commands - .spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye)) - .id(); - commands.entity(tracking_root).push_children(&[right, left]); -} - -pub fn action_set_system(action_sets: Res, session: Res) { - let mut active_action_sets = vec![]; - for i in &action_sets.0 { - active_action_sets.push(openxr::ActiveActionSet::new(i)); - } - //info!("action sets: {:#?}", action_sets.0.len()); - match session.sync_actions(&active_action_sets) { - Err(err) => { - warn!("{}", err); - } - _ => {} + for e in &tracking_root_query { + commands.entity(e).despawn_recursive(); } } +fn setup_xr_root( + mut commands: Commands, + tracking_root_query: Query>, +) { + if tracking_root_query.get_single().is_err() { + commands.spawn((SpatialBundle::default(), OpenXRTrackingRoot)); + } +} + +// pub fn action_set_system(action_sets: Res, session: Res) { +// let mut active_action_sets = vec![]; +// for i in &action_sets.0 { +// active_action_sets.push(openxr::ActiveActionSet::new(i)); +// } +// //info!("action sets: {:#?}", action_sets.0.len()); +// if let Err(err) = session.sync_actions(&active_action_sets) { +// warn!("{}", err); +// } +// } pub trait Vec2Conv { fn to_vec2(&self) -> Vec2; diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 50d38ed..f154d4c 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -1,15 +1,73 @@ +use crate::xr_init::{xr_only, XrCleanup, XrSetup}; use crate::xr_input::{QuatConv, Vec3Conv}; -use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; +use crate::{locate_views, xr_wait_frame, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; use bevy::ecs::system::lifetimeless::Read; use bevy::math::Vec3A; use bevy::prelude::*; -use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; -use bevy::render::extract_component::ExtractComponent; +use bevy::render::camera::{ + CameraProjection, CameraProjectionPlugin, CameraRenderGraph, RenderTarget, +}; +use bevy::render::extract_component::{ExtractComponent, ExtractComponentPlugin}; use bevy::render::primitives::Frustum; -use bevy::render::view::{ColorGrading, ExtractedView, VisibleEntities}; +use bevy::render::view::{ + update_frusta, ColorGrading, ExtractedView, VisibilitySystems, VisibleEntities, +}; +use bevy::transform::TransformSystem; use openxr::Fovf; +use super::trackers::{OpenXRLeftEye, OpenXRRightEye, OpenXRTracker, OpenXRTrackingRoot}; + +pub struct XrCameraPlugin; + +impl Plugin for XrCameraPlugin { + fn build(&self, app: &mut App) { + app.add_plugins(CameraProjectionPlugin::::default()); + app.add_systems( + PreUpdate, + xr_camera_head_sync + .run_if(xr_only()) + .after(xr_wait_frame) + .after(locate_views), + ); + // a little late latching + app.add_systems( + PostUpdate, + xr_camera_head_sync + .before(TransformSystem::TransformPropagate) + .run_if(xr_only()), + ); + app.add_systems( + PostUpdate, + update_frusta:: + .after(TransformSystem::TransformPropagate) + .before(VisibilitySystems::UpdatePerspectiveFrusta), + ); + app.add_systems(XrSetup, setup_xr_cameras); + app.add_systems(XrCleanup, cleanup_xr_cameras); + app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); + // app.add_plugins(ExtractComponentPlugin::::default()); + // app.add_plugins(ExtractComponentPlugin::::default()); + } +} + +// might be unnesesary since it should be parented to the root +fn cleanup_xr_cameras(mut commands: Commands, entities: Query>) { + for e in &entities { + commands.entity(e).despawn_recursive(); + } +} + +fn setup_xr_cameras(mut commands: Commands) { + commands.spawn(( + XrCameraBundle::new(Eye::Right), + OpenXRRightEye, + OpenXRTracker, + )); + commands.spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye, OpenXRTracker)); +} + #[derive(Bundle)] pub struct XrCamerasBundle { pub left: XrCameraBundle, @@ -42,13 +100,10 @@ pub struct XrCameraBundle { pub tonemapping: Tonemapping, pub dither: DebandDither, pub color_grading: ColorGrading, - pub xr_camera_type: XrCameraType, + pub xr_camera_type: XrCamera, } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)] -pub enum XrCameraType { - Xr(Eye), - Flatscreen, -} +pub struct XrCamera(Eye); #[derive(Component)] pub(super) struct TransformExtract; @@ -108,7 +163,7 @@ impl XrCameraBundle { tonemapping: Default::default(), dither: DebandDither::Enabled, color_grading: Default::default(), - xr_camera_type: XrCameraType::Xr(eye), + xr_camera_type: XrCamera(eye), } } } @@ -273,15 +328,11 @@ impl CameraProjection for XRProjection { pub fn xr_camera_head_sync( views: Res, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + mut query: Query<(&mut Transform, &XrCamera, &mut XRProjection)>, ) { //TODO calculate HMD position for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - XrCameraType::Xr(eye) => *eye as usize, - // I don't belive we need a flatscrenn cam, that's just a cam without this component - XrCameraType::Flatscreen => continue, - }; + let view_idx = camera_type.0 as usize; let view = match views.get(view_idx) { Some(views) => views, None => continue, @@ -291,25 +342,3 @@ pub fn xr_camera_head_sync( transform.translation = view.pose.position.to_vec3(); } } -pub fn xr_camera_head_sync_render( - views: Res, - mut query: Query<(&mut ExtractedView, &XrCameraType)>, -) { - //TODO calculate HMD position - for (mut transform, camera_type) in query.iter_mut() { - // let mut t = Transform::IDENTITY; - // let view_idx = match camera_type { - // XrCameraType::Xr(eye) => *eye as usize, - // // I don't belive we need a flatscrenn cam, that's just a cam without this component - // XrCameraType::Flatscreen => continue, - // }; - // let view = match views.get(view_idx) { - // Some(views) => views, - // None => continue, - // }; - // t.rotation = view.pose.orientation.to_quat(); - // t.translation = view.pose.position.to_vec3(); - info!("cam update"); - transform.transform = GlobalTransform::IDENTITY; - } -} From a95462b91d095e3e72a47de34395c0300d7e7625 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 20 Feb 2024 06:57:06 +0100 Subject: [PATCH 29/80] pre bevy 0.13 --- src/lib.rs | 19 +++++++++++-------- src/xr_input/hands/mod.rs | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 49c8508..9a3fb3b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -120,7 +120,6 @@ impl Plugin for OpenXrPlugin { xr_wait_frame.run_if(xr_only()), // xr_begin_frame.run_if(xr_only()), locate_views.run_if(xr_only()), - apply_deferred, ) .chain() @@ -306,23 +305,27 @@ fn xr_begin_frame(swapchain: Res) { } pub fn xr_wait_frame( - mut frame_state: ResMut, - mut frame_waiter: ResMut, - mut should_render: ResMut, - mut waited: ResMut, + world: &mut World, + // mut frame_state: ResMut, + // mut frame_waiter: ResMut, + // mut should_render: ResMut, + // mut waited: ResMut, ) { + let mut frame_waiter = world.get_resource_mut::().unwrap(); { let _span = info_span!("xr_wait_frame").entered(); - *frame_state = match frame_waiter.wait() { + + *world.get_resource_mut::().unwrap() = match frame_waiter.wait() { Ok(a) => a.into(), Err(e) => { warn!("error: {}", e); return; } }; + let should_render = world.get_resource::().unwrap().should_render; // frame_state.predicted_display_time = xr::Time::from_nanos(frame_state.predicted_display_time.as_nanos() + frame_state.predicted_display_period.as_nanos()); - **should_render = frame_state.should_render; - **waited = true; + **world.get_resource_mut::().unwrap() = should_render; + **world.get_resource_mut::().unwrap() = true; } } diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs index 8ab17d8..ea7a230 100644 --- a/src/xr_input/hands/mod.rs +++ b/src/xr_input/hands/mod.rs @@ -55,6 +55,7 @@ fn check_for_handtracking( .supports_hand_tracking(instance.system(FormFactor::HEAD_MOUNTED_DISPLAY).unwrap()) .is_ok_and(|v| v); if hands { + info!("handtracking!"); commands.insert_resource(HandTrackingData::new(&session).unwrap()); } else { commands.insert_resource(DisableHandTracking::Both); From df71c7931d54c9fa5e8b9e1e201c2c3b5f9171fc Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 20 Feb 2024 09:15:21 +0100 Subject: [PATCH 30/80] update to bevy 0.13. TODO: fix view weirdness and do a pass over most of xr_input to turn the modules into plugins --- Cargo.toml | 12 ++--- examples/android/Cargo.toml | 2 +- examples/android/src/lib.rs | 26 ++++++---- examples/demo/Cargo.toml | 5 +- examples/demo/src/lib.rs | 55 ++++++++------------ examples/demo/src/setup.rs | 17 +++---- examples/globe.rs | 16 +++--- examples/xr.rs | 25 +++++----- src/graphics/vulkan.rs | 13 +++-- src/lib.rs | 2 + src/xr_input/debug_gizmos.rs | 23 +++++---- src/xr_input/interactions.rs | 17 ++++--- src/xr_input/mod.rs | 2 +- src/xr_input/prototype_locomotion.rs | 30 ++++++----- src/xr_input/trackers.rs | 36 +++++++------ src/xr_input/xr_camera.rs | 75 +++++++++++++++------------- 16 files changed, 179 insertions(+), 177 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bb93ce7..847c1b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,12 +17,12 @@ members = ["examples/android", "examples/demo"] [dependencies] # anyhow = "1.0.75" ash = "0.37.3" -bevy = "0.12" +bevy = "0.13" futures-lite = "2.0.1" mint = "0.5.9" -wgpu = "0.17.1" -wgpu-core = { version = "0.17.1", features = ["vulkan"] } -wgpu-hal = "0.17.1" +wgpu = "0.19" +wgpu-core = { version = "0.19", features = ["vulkan"] } +wgpu-hal = "0.19" eyre = "0.6.11" [target.'cfg(windows)'.dependencies] @@ -46,7 +46,7 @@ ndk-context = "0.1" jni = "0.20" [dev-dependencies] -bevy = "0.12" +bevy = "0.13" bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } color-eyre = "0.6.2" @@ -58,4 +58,4 @@ path = "examples/xr.rs" debug = true [patch.crates-io] -ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +# ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } diff --git a/examples/android/Cargo.toml b/examples/android/Cargo.toml index 1c412bc..cafa8c2 100644 --- a/examples/android/Cargo.toml +++ b/examples/android/Cargo.toml @@ -13,7 +13,7 @@ crate-type = ["rlib", "cdylib"] [dependencies] bevy_oxr.path = "../.." -bevy = "0.12" +bevy = "0.13" openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = ["mint"] } # [profile.release] diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index a52a93c..9b1c664 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -17,7 +17,7 @@ use bevy_oxr::DefaultXrPlugins; #[bevy_main] fn main() { let mut xr_extensions = XrExtensions::default(); - xr_extensions.enable_fb_passthrough(); + // xr_extensions.enable_fb_passthrough(); xr_extensions.enable_hand_tracking(); App::new() .add_plugins(DefaultXrPlugins { @@ -31,7 +31,7 @@ fn main() { .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) .add_plugins(HandInputDebugRenderer) - .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup) + // .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup) .add_systems(Startup, setup) .add_systems( Update, @@ -57,21 +57,21 @@ fn setup( ) { // plane commands.spawn(PbrBundle { - mesh: meshes.add(shape::Plane::from_size(5.0).into()), - material: materials.add(Color::rgb(0.3, 0.5, 0.3).into()), + mesh: meshes.add(Plane3d::new(Vec3::Y)), + material: materials.add(StandardMaterial::from(Color::rgb(0.3, 0.5, 0.3))), ..default() }); // cube commands.spawn(PbrBundle { - mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })), - material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()), + mesh: meshes.add(Cuboid::from_size(Vec3::splat(0.1)).mesh()), + material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))), transform: Transform::from_xyz(0.0, 0.5, 0.0), ..default() }); // cube commands.spawn(PbrBundle { - mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })), - material: materials.add(Color::rgb(0.8, 0.0, 0.0).into()), + mesh: meshes.add(Mesh::from(Cuboid::from_size(Vec3::splat(0.1)))), + material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.0, 0.0))), transform: Transform::from_xyz(0.0, 0.5, 1.0), ..default() }); @@ -106,7 +106,7 @@ fn spawn_controllers_example(mut commands: Commands) { // TODO: make this a vr button fn toggle_passthrough( - keys: Res>, + keys: Res>, passthrough_state: Res, mut resume: EventWriter, mut pause: EventWriter, @@ -114,8 +114,12 @@ fn toggle_passthrough( if keys.just_pressed(KeyCode::Space) { match *passthrough_state { XrPassthroughState::Unsupported => {} - XrPassthroughState::Running => pause.send_default(), - XrPassthroughState::Paused => resume.send_default(), + XrPassthroughState::Running => { + pause.send_default(); + } + XrPassthroughState::Paused => { + resume.send_default(); + } } } } diff --git a/examples/demo/Cargo.toml b/examples/demo/Cargo.toml index 14e39de..00df137 100644 --- a/examples/demo/Cargo.toml +++ b/examples/demo/Cargo.toml @@ -9,9 +9,10 @@ crate-type = ["rlib", "cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bevy = "0.12" +bevy = "0.13" bevy_oxr.path = "../../" -bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } +# bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } +bevy_rapier3d = "0.25" color-eyre = "0.6.2" diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index 8f369f9..2874dc9 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -1,19 +1,13 @@ use std::{f32::consts::PI, ops::Mul, time::Duration}; use bevy::{ - diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}, - ecs::schedule::ScheduleLabel, - input::{keyboard::KeyCode, Input}, - log::info, - prelude::{ + diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}, ecs::schedule::ScheduleLabel, input::{keyboard::KeyCode, ButtonInput}, log::info, math::primitives::{Capsule3d, Cuboid}, prelude::{ bevy_main, default, shape, App, Assets, Color, Commands, Component, Entity, Event, EventReader, EventWriter, FixedUpdate, Gizmos, GlobalTransform, IntoSystemConfigs, IntoSystemSetConfigs, Mesh, PbrBundle, PostUpdate, Quat, Query, Res, ResMut, Resource, Schedule, SpatialBundle, StandardMaterial, Startup, Transform, Update, Vec3, Vec3Swizzles, With, Without, World, - }, - time::{Fixed, Time, Timer, TimerMode}, - transform::TransformSystem, + }, render::mesh::Meshable, time::{Fixed, Time, Timer, TimerMode}, transform::TransformSystem }; use bevy_oxr::{ graphics::{extensions::XrExtensions, XrAppInfo, XrPreferdBlendMode}, @@ -226,12 +220,8 @@ fn spawn_capsule( ) { commands.spawn(( PbrBundle { - mesh: meshes.add(Mesh::from(shape::Capsule { - radius: 0.033, - depth: 0.115, - ..default() - })), - material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()), + mesh: meshes.add(Capsule3d::new(0.033, 0.115).mesh()), + material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))), transform: Transform::from_xyz(0.0, 2.0, 0.0), ..default() }, @@ -376,7 +366,7 @@ fn update_physics_hands( &Hand, &mut Velocity, )>, - hand_query: Query<(&Transform, &HandBone, &Hand, Without)>, + hand_query: Query<(&Transform, &HandBone, &Hand), Without>, time: Res