From aae751bdabcecc929bf42ab2ec9c9bd20baafdfb Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 9 Jan 2024 20:34:10 +0100 Subject: [PATCH 01/25] changed vulkan init code --- Cargo.toml | 5 +- examples/demo/src/lib.rs | 4 - src/graphics/mod.rs | 13 ++-- src/graphics/vulkan.rs | 107 +++++++++++++++++---------- src/input.rs | 4 +- src/lib.rs | 39 +++++----- src/resource_macros.rs | 55 ++++++++++---- src/resources.rs | 21 +++++- src/xr_init.rs | 19 +++-- src/xr_input/actions.rs | 2 +- src/xr_input/debug_gizmos.rs | 2 +- src/xr_input/hands/hand_tracking.rs | 2 +- src/xr_input/oculus_touch.rs | 2 +- src/xr_input/prototype_locomotion.rs | 20 ++--- src/xr_input/trackers.rs | 2 - src/xr_input/xr_camera.rs | 14 ++-- 16 files changed, 191 insertions(+), 120 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2836170..2f6c6f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ force-link = ["openxr/linked"] members = ["examples/android", "examples/demo"] [dependencies] -anyhow = "1.0.75" +# anyhow = "1.0.75" ash = "0.37.3" bevy = "0.12" futures-lite = "2.0.1" @@ -23,6 +23,7 @@ mint = "0.5.9" wgpu = "0.17.1" wgpu-core = { version = "0.17.1", features = ["vulkan"] } wgpu-hal = "0.17.1" +eyre = "0.6.11" [target.'cfg(windows)'.dependencies] openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = [ @@ -42,8 +43,8 @@ openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features [dev-dependencies] bevy = "0.12" -color-eyre = "0.6.2" bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } +color-eyre = "0.6.2" [[example]] name = "xr" diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index 22529c7..ad92cd8 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -560,8 +560,6 @@ fn request_cube_spawn( ) { timer.0.tick(time.delta()); if timer.0.finished() { - //lock frame - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get controller triggers @@ -628,8 +626,6 @@ fn prototype_interaction_input( >, action_sets: Res, ) { - //lock frame - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get controller triggers diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 0aa3dfc..49baa88 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -7,8 +7,8 @@ use wgpu::Instance; use crate::input::XrInput; use crate::resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, + XrSessionRunning, XrSwapchain, XrViews, XrFrameWaiter, }; use openxr as xr; @@ -44,7 +44,7 @@ pub fn initialize_xr_graphics( reqeusted_extensions: XrExtensions, prefered_blend_mode: XrPreferdBlendMode, app_info: XrAppInfo, -) -> anyhow::Result<( +) -> eyre::Result<( RenderDevice, RenderQueue, RenderAdapterInfo, @@ -62,13 +62,14 @@ pub fn initialize_xr_graphics( XrViews, XrFrameState, )> { - vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info) + // vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info) + todo!() } -pub fn xr_entry() -> anyhow::Result { +pub fn xr_entry() -> eyre::Result { #[cfg(windows)] let entry = Ok(xr::Entry::linked()); #[cfg(not(windows))] - let entry = unsafe { xr::Entry::load().map_err(|e| anyhow::anyhow!(e)) }; + let entry = unsafe { xr::Entry::load().map_err(|e| eyre::eyre!(e)) }; entry } diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index ba9814e..1411491 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -1,52 +1,45 @@ use std::ffi::{c_void, CString}; use std::sync::atomic::AtomicBool; -use std::sync::{Arc, Mutex}; +use std::sync::Mutex; -use anyhow::Context; +// use anyhow::Context; use ash::vk::{self, Handle}; use bevy::math::uvec2; use bevy::prelude::*; use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}; use bevy::window::RawHandleWrapper; +use eyre::{Context, ContextCompat}; use openxr as xr; use wgpu::Instance; +use wgpu_hal::{api::Vulkan as V, Api}; use xr::EnvironmentBlendMode; use crate::graphics::extensions::XrExtensions; use crate::input::XrInput; use crate::resources::{ - Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, - XrInstance, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, + OXrSessionSetupInfo, Swapchain, SwapchainInner, VulkanOXrSessionSetupInfo, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, + XrSession, XrSessionRunning, XrSwapchain, XrViews, }; use crate::VIEW_TYPE; use super::{XrAppInfo, XrPreferdBlendMode}; -pub fn initialize_xr_graphics( +pub fn initialize_xr_instance( window: Option, reqeusted_extensions: XrExtensions, prefered_blend_mode: XrPreferdBlendMode, app_info: XrAppInfo, -) -> anyhow::Result<( +) -> eyre::Result<( + XrInstance, + OXrSessionSetupInfo, + XrEnvironmentBlendMode, RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter, Instance, - XrInstance, - XrSession, - XrEnvironmentBlendMode, - XrResolution, - XrFormat, - XrSessionRunning, - XrFrameWaiter, - XrSwapchain, - XrInput, - XrViews, - XrFrameState, )> { - use wgpu_hal::{api::Vulkan as V, Api}; - let xr_entry = super::xr_entry()?; #[cfg(target_os = "android")] @@ -108,7 +101,6 @@ pub fn initialize_xr_graphics( _ => EnvironmentBlendMode::OPAQUE, }; - #[cfg(not(target_os = "android"))] let vk_target_version = vk::make_api_version(0, 1, 2, 0); #[cfg(not(target_os = "android"))] @@ -296,22 +288,66 @@ pub fn initialize_xr_graphics( None, ) }?; + Ok(( + xr_instance.into(), + OXrSessionSetupInfo::Vulkan(VulkanOXrSessionSetupInfo { + device_ptr: vk_device_ptr, + physical_device_ptr: vk_physical_device_ptr, + vk_instance_ptr, + queue_family_index, + xr_system_id, + }), + blend_mode.into(), + wgpu_device.into(), + RenderQueue(wgpu_queue.into()), + RenderAdapterInfo(wgpu_adapter.get_info()), + RenderAdapter(wgpu_adapter.into()), + wgpu_instance.into(), + )) +} +pub fn initialize_xr_graphics( + window: Option, + ptrs: &OXrSessionSetupInfo, + xr_instance: &XrInstance, + render_device: &RenderDevice, + render_adapter: &RenderAdapter, + wgpu_instance: &Instance, +) -> eyre::Result<( + XrInstance, + XrSession, + XrResolution, + XrFormat, + XrSessionRunning, + XrFrameWaiter, + XrSwapchain, + XrInput, + XrViews, + XrFrameState, +)> { + let wgpu_device = render_device.wgpu_device(); + let wgpu_adapter = &render_adapter.0; + + #[allow(unreachable_patterns)] + let setup_info = match ptrs { + OXrSessionSetupInfo::Vulkan(v) => v, + _ => eyre::bail!("Wrong Graphics Api"), + }; let (session, frame_wait, frame_stream) = unsafe { xr_instance.create_session::( - xr_system_id, + xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY)?, &xr::vulkan::SessionCreateInfo { - instance: vk_instance_ptr, - physical_device: vk_physical_device_ptr, - device: vk_device_ptr, - queue_family_index, + instance: setup_info.vk_instance_ptr, + physical_device: setup_info.physical_device_ptr, + device: setup_info.device_ptr, + queue_family_index: setup_info.queue_family_index, queue_index: 0, }, ) }?; - let views = xr_instance.enumerate_view_configuration_views(xr_system_id, VIEW_TYPE)?; - + let views = + xr_instance.enumerate_view_configuration_views(setup_info.xr_system_id, VIEW_TYPE)?; let surface = window.map(|wrapper| unsafe { // SAFETY: Plugins should be set up on the main thread. let handle = wrapper.get_handle(); @@ -346,6 +382,7 @@ pub fn initialize_xr_graphics( mip_count: 1, }) .unwrap(); + let images = handle.enumerate_images().unwrap(); let buffers = images @@ -399,18 +436,12 @@ pub fn initialize_xr_graphics( .collect(); Ok(( - wgpu_device.into(), - RenderQueue(Arc::new(wgpu_queue)), - RenderAdapterInfo(wgpu_adapter.get_info()), - RenderAdapter(Arc::new(wgpu_adapter)), - wgpu_instance, xr_instance.clone().into(), session.clone().into_any_graphics().into(), - blend_mode.into(), resolution.into(), swapchain_format.into(), AtomicBool::new(false).into(), - Mutex::new(frame_wait).into(), + frame_wait.into(), Swapchain::Vulkan(SwapchainInner { stream: Mutex::new(frame_stream), handle: Mutex::new(handle), @@ -418,13 +449,13 @@ pub fn initialize_xr_graphics( image_index: Mutex::new(0), }) .into(), - XrInput::new(xr_instance, session.into_any_graphics())?, - Mutex::default().into(), - Mutex::new(xr::FrameState { + XrInput::new(xr_instance, &session.into_any_graphics())?, + Vec::default().into(), + xr::FrameState { predicted_display_time: xr::Time::from_nanos(1), predicted_display_period: xr::Duration::from_nanos(1), should_render: true, - }) + } .into(), )) } diff --git a/src/input.rs b/src/input.rs index 561969f..cf81c0e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -16,8 +16,8 @@ pub struct XrInput { impl XrInput { pub fn new( - instance: xr::Instance, - session: xr::Session, + instance: &xr::Instance, + session: &xr::Session, // frame_state: &FrameState, ) -> xr::Result { // let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 9a77e87..6f9a591 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,7 +26,7 @@ use input::XrInput; use openxr as xr; // use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; use resources::*; -use xr::FormFactor; +use xr::{FormFactor, FrameWaiter}; use xr_init::{xr_only, XrEnableStatus, XrRenderData}; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; @@ -57,7 +57,7 @@ pub struct FutureXrResources( XrResolution, XrFormat, XrSessionRunning, - XrFrameWaiter, + // XrFrameWaiter, XrSwapchain, XrInput, XrViews, @@ -110,7 +110,7 @@ impl Plugin for OpenXrPlugin { app.insert_resource(resolution.clone()); app.insert_resource(format.clone()); app.insert_resource(session_running.clone()); - app.insert_resource(frame_waiter.clone()); + app.insert_resource(frame_waiter); app.insert_resource(swapchain.clone()); app.insert_resource(input.clone()); app.insert_resource(views.clone()); @@ -122,7 +122,6 @@ impl Plugin for OpenXrPlugin { xr_resolution: resolution, xr_format: format, xr_session_running: session_running, - xr_frame_waiter: frame_waiter, xr_swapchain: swapchain, xr_input: input, xr_views: views, @@ -221,7 +220,7 @@ impl Plugin for OpenXrPlugin { render_app.insert_resource(data.xr_resolution.clone()); render_app.insert_resource(data.xr_format.clone()); render_app.insert_resource(data.xr_session_running.clone()); - render_app.insert_resource(data.xr_frame_waiter.clone()); + // render_app.insert_resource(data.xr_frame_waiter.clone()); render_app.insert_resource(data.xr_swapchain.clone()); render_app.insert_resource(data.xr_input.clone()); render_app.insert_resource(data.xr_views.clone()); @@ -286,10 +285,10 @@ pub fn xr_begin_frame( instance: Res, session: Res, session_running: Res, - frame_state: Res, - frame_waiter: Res, + mut frame_state: ResMut, + mut frame_waiter: ResMut, swapchain: Res, - views: Res, + mut views: ResMut, input: Res, mut app_exit: EventWriter, ) { @@ -303,6 +302,7 @@ pub fn xr_begin_frame( // find quit messages! info!("entered XR state {:?}", e.state()); match e.state() { + // xr_frame_waiter: frame_waiter, xr::SessionState::READY => { session.begin(VIEW_TYPE).unwrap(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); @@ -316,6 +316,7 @@ pub fn xr_begin_frame( app_exit.send(AppExit); return; } + _ => {} } } @@ -329,8 +330,8 @@ pub fn xr_begin_frame( } { let _span = info_span!("xr_wait_frame").entered(); - *frame_state.lock().unwrap() = match frame_waiter.lock().unwrap().wait() { - Ok(a) => a, + *frame_state = match frame_waiter.wait() { + Ok(a) => a.into(), Err(e) => { warn!("error: {}", e); return; @@ -343,12 +344,8 @@ pub fn xr_begin_frame( } { let _span = info_span!("xr_locate_views").entered(); - *views.lock().unwrap() = session - .locate_views( - VIEW_TYPE, - frame_state.lock().unwrap().predicted_display_time, - &input.stage, - ) + **views = session + .locate_views(VIEW_TYPE, frame_state.predicted_display_time, &input.stage) .unwrap() .1; } @@ -402,8 +399,8 @@ pub fn end_frame( { let _span = info_span!("xr_end_frame").entered(); let result = swapchain.end( - xr_frame_state.lock().unwrap().predicted_display_time, - &views.lock().unwrap(), + xr_frame_state.predicted_display_time, + &views, &input.stage, **resolution, **environment_blend_mode, @@ -417,15 +414,15 @@ pub fn end_frame( } pub fn locate_views( - views: Res, + mut views: ResMut, input: Res, session: Res, xr_frame_state: Res, ) { let _span = info_span!("xr_locate_views").entered(); - *views.lock().unwrap() = match session.locate_views( + **views = match session.locate_views( VIEW_TYPE, - xr_frame_state.lock().unwrap().predicted_display_time, + xr_frame_state.predicted_display_time, &input.stage, ) { Ok(this) => this, diff --git a/src/resource_macros.rs b/src/resource_macros.rs index 5f4df4e..933ce20 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -1,7 +1,7 @@ #[macro_export] macro_rules! xr_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource)] + #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] pub struct $wrapper_type($xr_type); impl $wrapper_type { @@ -10,13 +10,13 @@ macro_rules! xr_resource_wrapper { } } - impl std::ops::Deref for $wrapper_type { - type Target = $xr_type; - - fn deref(&self) -> &Self::Target { - &self.0 - } - } + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // &self.0 + // } + // } impl From<$xr_type> for $wrapper_type { fn from(value: $xr_type) -> Self { @@ -29,7 +29,7 @@ macro_rules! xr_resource_wrapper { #[macro_export] macro_rules! xr_arc_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource)] + #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] pub struct $wrapper_type(std::sync::Arc<$xr_type>); impl $wrapper_type { @@ -38,13 +38,41 @@ macro_rules! xr_arc_resource_wrapper { } } - impl std::ops::Deref for $wrapper_type { - type Target = $xr_type; + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // self.0.as_ref() + // } + // } - fn deref(&self) -> &Self::Target { - self.0.as_ref() + impl From<$xr_type> for $wrapper_type { + fn from(value: $xr_type) -> Self { + Self::new(value) } } + }; +} + +#[macro_export] +macro_rules! xr_no_clone_resource_wrapper { + ($wrapper_type:ident, $xr_type:ty) => { + #[derive(bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] + pub struct $wrapper_type($xr_type); + + impl $wrapper_type { + pub fn new(value: $xr_type) -> Self { + Self(value) + } + } + + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // &self.0 + // } + // } impl From<$xr_type> for $wrapper_type { fn from(value: $xr_type) -> Self { @@ -55,4 +83,5 @@ macro_rules! xr_arc_resource_wrapper { } pub use xr_arc_resource_wrapper; +pub use xr_no_clone_resource_wrapper; pub use xr_resource_wrapper; diff --git a/src/resources.rs b/src/resources.rs index a798955..62f2855 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -1,3 +1,4 @@ +use std::ffi::c_void; use std::sync::atomic::AtomicBool; use std::sync::Mutex; @@ -5,17 +6,31 @@ use std::sync::Mutex; use crate::resource_macros::*; use bevy::prelude::*; use openxr as xr; +use xr::ViewConfigurationView; xr_resource_wrapper!(XrInstance, xr::Instance); xr_resource_wrapper!(XrSession, xr::Session); xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); xr_resource_wrapper!(XrResolution, UVec2); xr_resource_wrapper!(XrFormat, wgpu::TextureFormat); +xr_resource_wrapper!(XrFrameState, xr::FrameState); +xr_resource_wrapper!(XrViews, Vec); xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool); -xr_arc_resource_wrapper!(XrFrameWaiter, Mutex); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); -xr_arc_resource_wrapper!(XrFrameState, Mutex); -xr_arc_resource_wrapper!(XrViews, Mutex>); +xr_no_clone_resource_wrapper!(XrFrameWaiter, xr::FrameWaiter); + +pub(crate) struct VulkanOXrSessionSetupInfo { + pub(crate) device_ptr: *const c_void, + pub(crate) physical_device_ptr: *const c_void, + pub(crate) vk_instance_ptr: *const c_void, + pub(crate) queue_family_index: u32, + pub(crate) xr_system_id: xr::SystemId, + // pub(crate) swapchain_create_info: xr::SwapchainCreateInfo +} + +pub(crate) enum OXrSessionSetupInfo { + Vulkan(VulkanOXrSessionSetupInfo), +} pub enum Swapchain { Vulkan(SwapchainInner), diff --git a/src/xr_init.rs b/src/xr_init.rs index 499e7ce..f60b0ba 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -11,16 +11,16 @@ use bevy::{ renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}, settings::WgpuSettings, }, - window::RawHandleWrapper, + window::{RawHandleWrapper, PrimaryWindow}, }; use wgpu::Instance; use crate::{ input::XrInput, resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, - }, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, + XrSessionRunning, XrSwapchain, XrViews, + }, graphics, }; #[derive(Resource, Clone)] @@ -40,7 +40,7 @@ pub struct XrRenderData { pub xr_resolution: XrResolution, pub xr_format: XrFormat, pub xr_session_running: XrSessionRunning, - pub xr_frame_waiter: XrFrameWaiter, + // pub xr_frame_waiter: XrFrameWaiter, pub xr_swapchain: XrSwapchain, pub xr_input: XrInput, pub xr_views: XrViews, @@ -174,6 +174,14 @@ pub fn update_xr_stuff(world: &mut World) { world.run_schedule(XrPostRenderUpdate); } +fn setup_xr_graphics() { + +} + +fn enable_xr( + +) {} + // fn handle_xr_enable_requests( // primary_window: Query<&RawHandleWrapper, With>, // mut commands: Commands, @@ -183,7 +191,6 @@ pub fn update_xr_stuff(world: &mut World) { // // Just to force this system onto the main thread because of unsafe code // let _ = on_main; // -// commands.insert_resource(XrEnableStatus::Waiting); // let (creation_data, xr_data) = match next_state.into_inner() { // XrNextEnabledState::Enabled => { // let ( diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 9e71511..748968d 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -206,7 +206,7 @@ impl SetupActionSet { for binding in bindings { self.actions .get_mut(binding.action) - .ok_or(anyhow::anyhow!("Missing Action: {}", binding.action)) + .ok_or(eyre::eyre!("Missing Action: {}", binding.action)) .unwrap() .bindings .entry(device_path) diff --git a/src/xr_input/debug_gizmos.rs b/src/xr_input/debug_gizmos.rs index c67d900..ff93ef8 100644 --- a/src/xr_input/debug_gizmos.rs +++ b/src/xr_input/debug_gizmos.rs @@ -91,7 +91,7 @@ pub fn draw_gizmos( // } // } //lock frame - let frame_state = *frame_state.lock().unwrap(); + // let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); let root = tracking_root_query.get_single(); diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index 1af55df..fd8c920 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -87,7 +87,7 @@ impl<'a> HandTrackingRef<'a> { Hand::Left => &self.tracking.left_hand, Hand::Right => &self.tracking.right_hand, }, - self.frame_state.lock().unwrap().predicted_display_time, + self.frame_state.predicted_display_time, ) .unwrap() .map(|joints| { diff --git a/src/xr_input/oculus_touch.rs b/src/xr_input/oculus_touch.rs index 6f405f8..27111c4 100644 --- a/src/xr_input/oculus_touch.rs +++ b/src/xr_input/oculus_touch.rs @@ -371,7 +371,7 @@ pub struct OculusController { pub aim_space: Option>, } impl OculusController { - pub fn new(mut action_sets: ResMut) -> anyhow::Result { + pub fn new(mut action_sets: ResMut) -> eyre::Result { let action_set = action_sets.add_action_set("oculus_input", "Oculus Touch Controller Input".into(), 0); action_set.new_action( diff --git a/src/xr_input/prototype_locomotion.rs b/src/xr_input/prototype_locomotion.rs index 52bf355..791ee04 100644 --- a/src/xr_input/prototype_locomotion.rs +++ b/src/xr_input/prototype_locomotion.rs @@ -70,17 +70,13 @@ pub fn proto_locomotion( config_option: Option>, action_sets: Res, ) { - match config_option { - Some(_) => (), + let mut config = match config_option { + Some(c) => c, None => { info!("no locomotion config"); return; } - } - //i hate this but im too tired to think - let mut config = config_option.unwrap(); - //lock frame - let frame_state = *frame_state.lock().unwrap(); + }; //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); let root = tracking_root_query.get_single_mut(); @@ -92,8 +88,7 @@ pub fn proto_locomotion( let reference_quat; match config.locomotion_type { LocomotionType::Head => { - let v = views.lock().unwrap(); - let views = v.get(0); + let views = views.first(); match views { Some(view) => { reference_quat = view.pose.orientation.to_quat(); @@ -127,8 +122,7 @@ pub fn proto_locomotion( rot_input * config.smooth_rotation_speed * time.delta_seconds(), ); //apply rotation - let v = views.lock().unwrap(); - let views = v.get(0); + let views = views.first(); match views { Some(view) => { let mut hmd_translation = view.pose.position.to_vec3(); @@ -159,8 +153,8 @@ pub fn proto_locomotion( let smoth_rot = Quat::from_axis_angle(position.0.up(), config.snap_angle * dir); //apply rotation - let v = views.lock().unwrap(); - let views = v.get(0); + let v = views; + let views = v.first(); match views { Some(view) => { let mut hmd_translation = view.pose.position.to_vec3(); diff --git a/src/xr_input/trackers.rs b/src/xr_input/trackers.rs index a57e8a8..eea28c1 100644 --- a/src/xr_input/trackers.rs +++ b/src/xr_input/trackers.rs @@ -66,8 +66,6 @@ pub fn update_open_xr_controllers( session: Res, action_sets: Res, ) { - //lock dat frame? - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get left controller diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 64528f8..210feaa 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -241,22 +241,24 @@ impl CameraProjection for XRProjection { pub fn xr_camera_head_sync( views: ResMut, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, ) { - let mut f = || -> Option<()> { + fn f( + views: ResMut, + mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + ) -> Option<()> { //TODO calculate HMD position for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { let view_idx = match camera_type { XrCameraType::Xr(eye) => *eye as usize, XrCameraType::Flatscreen => return None, }; - let v = views.lock().unwrap(); - let view = v.get(view_idx)?; + let view = views.get(view_idx)?; xr_projection.fov = view.fov; transform.rotation = view.pose.orientation.to_quat(); transform.translation = view.pose.position.to_vec3(); } Some(()) - }; - let _ = f(); + } + let _ = f(views, query); } From cbf9485bb78456f83d935f5b8924941fd3913648 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Sun, 14 Jan 2024 00:15:14 +0100 Subject: [PATCH 02/25] working --- examples/demo/src/lib.rs | 1 - examples/globe.rs | 3 +- examples/xr.rs | 2 - src/graphics/mod.rs | 133 +++++++++++++++++++++++++++----- src/graphics/vulkan.rs | 4 +- src/input.rs | 4 +- src/lib.rs | 158 ++++++++++++--------------------------- src/resource_macros.rs | 16 +++- src/resources.rs | 20 ++++- src/xr_init.rs | 38 +++++----- src/xr_input/trackers.rs | 9 ++- 11 files changed, 224 insertions(+), 164 deletions(-) diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index ad92cd8..2867a23 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -50,7 +50,6 @@ fn input_stuff( match status.into_inner() { XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable), XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable), - XrEnableStatus::Waiting => (), } } } diff --git a/examples/globe.rs b/examples/globe.rs index cf58bea..192916b 100644 --- a/examples/globe.rs +++ b/examples/globe.rs @@ -172,8 +172,7 @@ fn pull_to_ground( let (globe_pos, globe) = globe.single(); // Get player position (position of playground + position within playground) - let v = views.lock().unwrap(); - let Some(view) = v.get(0) else { return }; + let Some(view) = views.first() else { return }; let mut hmd_translation = view.pose.position.to_vec3(); hmd_translation.y = 0.0; let local = root.translation; diff --git a/examples/xr.rs b/examples/xr.rs index b38d79a..6f8d2de 100644 --- a/examples/xr.rs +++ b/examples/xr.rs @@ -167,8 +167,6 @@ fn prototype_interaction_input( >, action_sets: Res, ) { - //lock frame - let frame_state = *frame_state.lock().unwrap(); //get controller let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets); //get controller triggers diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 49baa88..24e80ff 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -1,14 +1,19 @@ pub mod extensions; mod vulkan; -use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}; -use bevy::window::RawHandleWrapper; +use bevy::ecs::query::With; +use bevy::ecs::system::{Query, SystemState}; +use bevy::ecs::world::World; +use bevy::render::renderer::{ + RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, +}; +use bevy::window::{PrimaryWindow, RawHandleWrapper}; use wgpu::Instance; use crate::input::XrInput; use crate::resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, - XrSessionRunning, XrSwapchain, XrViews, XrFrameWaiter, + OXrSessionSetupInfo, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, + XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }; use openxr as xr; @@ -39,20 +44,15 @@ impl Default for XrAppInfo { } } -pub fn initialize_xr_graphics( +pub fn start_xr_session( window: Option, - reqeusted_extensions: XrExtensions, - prefered_blend_mode: XrPreferdBlendMode, - app_info: XrAppInfo, + session_setup_data: &OXrSessionSetupInfo, + xr_instance: &XrInstance, + render_device: &RenderDevice, + render_adapter: &RenderAdapter, + wgpu_instance: &Instance, ) -> eyre::Result<( - RenderDevice, - RenderQueue, - RenderAdapterInfo, - RenderAdapter, - Instance, - XrInstance, XrSession, - XrEnvironmentBlendMode, XrResolution, XrFormat, XrSessionRunning, @@ -62,8 +62,107 @@ pub fn initialize_xr_graphics( XrViews, XrFrameState, )> { - // vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info) - todo!() + vulkan::start_xr_session( + window, + session_setup_data, + xr_instance, + render_device, + render_adapter, + wgpu_instance, + ) +} +pub fn initialize_xr_instance( + window: Option, + reqeusted_extensions: XrExtensions, + prefered_blend_mode: XrPreferdBlendMode, + app_info: XrAppInfo, +) -> eyre::Result<( + XrInstance, + OXrSessionSetupInfo, + XrEnvironmentBlendMode, + RenderDevice, + RenderQueue, + RenderAdapterInfo, + RenderAdapter, + Instance, +)> { + vulkan::initialize_xr_instance(window, reqeusted_extensions, prefered_blend_mode, app_info) +} + +pub fn try_full_init( + world: &mut World, + reqeusted_extensions: XrExtensions, + prefered_blend_mode: XrPreferdBlendMode, + app_info: XrAppInfo, +) -> eyre::Result<( + RenderDevice, + RenderQueue, + RenderAdapterInfo, + RenderAdapter, + RenderInstance, +)> { + let mut system_state: SystemState>> = + SystemState::new(world); + let primary_window = system_state.get(&world).get_single().ok().cloned(); + let ( + xr_instance, + setup_info, + blend_mode, + render_device, + render_queue, + render_adapter_info, + render_adapter, + wgpu_instance, + ) = initialize_xr_instance( + primary_window.clone(), + reqeusted_extensions, + prefered_blend_mode, + app_info, + )?; + world.insert_resource(xr_instance); + world.insert_non_send_resource(setup_info); + // TODO: move BlendMode the session init? + world.insert_resource(blend_mode); + let setup_info = world + .get_non_send_resource::() + .unwrap(); + let xr_instance = world.get_resource::().unwrap(); + + let ( + xr_session, + xr_resolution, + xr_format, + xr_session_running, + xr_frame_waiter, + xr_swapchain, + xr_input, + xr_views, + xr_frame_state, + ) = start_xr_session( + primary_window, + setup_info, + xr_instance, + &render_device, + &render_adapter, + &wgpu_instance, + )?; + world.insert_resource(xr_session); + world.insert_resource(xr_resolution); + world.insert_resource(xr_format); + world.insert_resource(xr_session_running); + world.insert_resource(xr_frame_waiter); + world.insert_resource(xr_swapchain); + world.insert_resource(xr_input); + world.insert_resource(xr_views); + world.insert_resource(xr_frame_state); + + Ok(( + render_device, + render_queue, + render_adapter_info, + render_adapter, + RenderInstance(wgpu_instance.into()), + )) } pub fn xr_entry() -> eyre::Result { diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index 1411491..ac0948d 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -306,7 +306,7 @@ pub fn initialize_xr_instance( )) } -pub fn initialize_xr_graphics( +pub fn start_xr_session( window: Option, ptrs: &OXrSessionSetupInfo, xr_instance: &XrInstance, @@ -314,7 +314,6 @@ pub fn initialize_xr_graphics( render_adapter: &RenderAdapter, wgpu_instance: &Instance, ) -> eyre::Result<( - XrInstance, XrSession, XrResolution, XrFormat, @@ -436,7 +435,6 @@ pub fn initialize_xr_graphics( .collect(); Ok(( - xr_instance.clone().into(), session.clone().into_any_graphics().into(), resolution.into(), swapchain_format.into(), diff --git a/src/input.rs b/src/input.rs index cf81c0e..8a98a7e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1,10 +1,10 @@ use std::sync::Arc; -use bevy::prelude::*; +use bevy::{prelude::*, render::extract_resource::ExtractResource}; use openxr as xr; use xr::{FrameState, FrameWaiter, ViewConfigurationType}; -#[derive(Clone, Resource)] +#[derive(Clone, Resource, ExtractResource)] pub struct XrInput { //pub action_set: xr::ActionSet, //pub hand_pose: xr::Action, diff --git a/src/lib.rs b/src/lib.rs index 6f9a591..ea1e496 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,11 +15,13 @@ use bevy::app::{AppExit, PluginGroupBuilder}; use bevy::ecs::system::SystemState; use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; +use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; +use eyre::anyhow; use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; use input::XrInput; @@ -57,7 +59,6 @@ pub struct FutureXrResources( XrResolution, XrFormat, XrSessionRunning, - // XrFrameWaiter, XrSwapchain, XrInput, XrViews, @@ -66,68 +67,20 @@ pub struct FutureXrResources( >, >, ); -// fn mr_test(mut commands: Commands, passthrough_layer: Option>) { -// commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); -// } impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { - let mut system_state: SystemState>> = - SystemState::new(&mut app.world); - let primary_window = system_state.get(&app.world).get_single().ok().cloned(); - #[cfg(not(target_arch = "wasm32"))] - match graphics::initialize_xr_graphics( - primary_window.clone(), + match graphics::try_full_init( + &mut app.world, self.reqeusted_extensions.clone(), self.prefered_blend_mode, self.app_info.clone(), ) { - Ok(( - device, - queue, - adapter_info, - render_adapter, - instance, - xr_instance, - session, - blend_mode, - resolution, - format, - session_running, - frame_waiter, - swapchain, - input, - views, - frame_state, - )) => { - // std::thread::sleep(Duration::from_secs(5)); + Ok((device, queue, adapter_info, render_adapter, instance)) => { debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); debug!("Configured wgpu adapter Features: {:#?}", device.features()); - app.insert_resource(xr_instance.clone()); - app.insert_resource(session.clone()); - app.insert_resource(blend_mode.clone()); - app.insert_resource(resolution.clone()); - app.insert_resource(format.clone()); - app.insert_resource(session_running.clone()); - app.insert_resource(frame_waiter); - app.insert_resource(swapchain.clone()); - app.insert_resource(input.clone()); - app.insert_resource(views.clone()); - app.insert_resource(frame_state.clone()); - let xr_data = XrRenderData { - xr_instance, - xr_session: session, - xr_blend_mode: blend_mode, - xr_resolution: resolution, - xr_format: format, - xr_session_running: session_running, - xr_swapchain: swapchain, - xr_input: input, - xr_views: views, - xr_frame_state: frame_state, - }; - app.insert_resource(xr_data); + warn!("Starting in Xr"); app.insert_resource(ActionSets(vec![])); app.add_plugins(RenderPlugin { render_creation: RenderCreation::Manual( @@ -135,14 +88,16 @@ impl Plugin for OpenXrPlugin { queue, adapter_info, render_adapter, - RenderInstance(Arc::new(instance)), + instance, ), }); + app.add_plugins(ExtractResourcePlugin::::default()); app.insert_resource(XrEnableStatus::Enabled); } Err(err) => { warn!("OpenXR Failed to initialize: {}", err); app.add_plugins(RenderPlugin::default()); + app.add_plugins(ExtractResourcePlugin::::default()); app.insert_resource(XrEnableStatus::Disabled); } } @@ -154,58 +109,48 @@ impl Plugin for OpenXrPlugin { } } - fn ready(&self, app: &App) -> bool { - app.world - .get_resource::() - .map(|frr| *frr != XrEnableStatus::Waiting) - .unwrap_or(true) - } - fn finish(&self, app: &mut App) { // TODO: Split this up into the indevidual resources - if let Some(data) = app.world.get_resource::().cloned() { - let hands = data.xr_instance.exts().ext_hand_tracking.is_some() - && data - .xr_instance + if app.world.get_resource::() == Some(&XrEnableStatus::Enabled) { + warn!("finished xr init"); + let xr_instance = app + .world + .get_resource::() + .expect("should exist"); + let xr_session = app.world.get_resource::().expect("should exist"); + let hands = xr_instance.exts().ext_hand_tracking.is_some() + && xr_instance .supports_hand_tracking( - data.xr_instance + xr_instance .system(FormFactor::HEAD_MOUNTED_DISPLAY) .unwrap(), ) .is_ok_and(|v| v); if hands { - app.insert_resource(HandTrackingData::new(&data.xr_session).unwrap()); + app.insert_resource(HandTrackingData::new(xr_session).unwrap()); } else { app.insert_resource(DisableHandTracking::Both); } - // let passthrough = data.xr_instance.exts().fb_passthrough.is_some() - // && supports_passthrough( - // &data.xr_instance, - // data.xr_instance - // .system(FormFactor::HEAD_MOUNTED_DISPLAY) - // .unwrap(), - // ) - // .is_ok_and(|v| v); - // if passthrough { - // info!("Passthrough!"); - // let (pl, p) = start_passthrough(&data); - // app.insert_resource(pl); - // app.insert_resource(p); - // // if !app.world.contains_resource::() { - // // info!("ClearColor!"); - // // } - // } + let xr_swapchain = app + .world + .get_resource::() + .expect("should exist"); + let xr_resolution = app + .world + .get_resource::() + .expect("should exist"); + let xr_format = app.world.get_resource::().expect("should exist"); - let (left, right) = data.xr_swapchain.get_render_views(); + let (left, right) = xr_swapchain.get_render_views(); let left = ManualTextureView { texture_view: left.into(), - size: *data.xr_resolution, - format: *data.xr_format, + size: **xr_resolution, + format: **xr_format, }; let right = ManualTextureView { texture_view: right.into(), - size: *data.xr_resolution, - format: *data.xr_format, + size: **xr_resolution, + format: **xr_format, }; app.add_systems(PreUpdate, xr_begin_frame.run_if(xr_only())); let mut manual_texture_views = app.world.resource_mut::(); @@ -213,19 +158,6 @@ impl Plugin for OpenXrPlugin { manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); drop(manual_texture_views); let render_app = app.sub_app_mut(RenderApp); - - render_app.insert_resource(data.xr_instance.clone()); - render_app.insert_resource(data.xr_session.clone()); - render_app.insert_resource(data.xr_blend_mode.clone()); - render_app.insert_resource(data.xr_resolution.clone()); - render_app.insert_resource(data.xr_format.clone()); - render_app.insert_resource(data.xr_session_running.clone()); - // render_app.insert_resource(data.xr_frame_waiter.clone()); - render_app.insert_resource(data.xr_swapchain.clone()); - render_app.insert_resource(data.xr_input.clone()); - render_app.insert_resource(data.xr_views.clone()); - render_app.insert_resource(data.xr_frame_state.clone()); - render_app.insert_resource(XrEnableStatus::Enabled); render_app.add_systems( Render, ( @@ -262,6 +194,8 @@ impl PluginGroup for DefaultXrPlugins { .add_before::(RenderRestartPlugin) .add(HandEmulationPlugin) .add(HandTrackingPlugin) + .add(XrResourcePlugin) + // .add(xr_init::RenderRestartPlugin) .set(WindowPlugin { #[cfg(not(target_os = "android"))] primary_window: Some(Window { @@ -302,7 +236,6 @@ pub fn xr_begin_frame( // find quit messages! info!("entered XR state {:?}", e.state()); match e.state() { - // xr_frame_waiter: frame_waiter, xr::SessionState::READY => { session.begin(VIEW_TYPE).unwrap(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); @@ -310,7 +243,6 @@ pub fn xr_begin_frame( xr::SessionState::STOPPING => { session.end().unwrap(); session_running.store(false, std::sync::atomic::Ordering::Relaxed); - app_exit.send(AppExit); } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { app_exit.send(AppExit); @@ -384,14 +316,20 @@ pub fn post_frame( } pub fn end_frame( - xr_frame_state: Res, - views: Res, - input: Res, - swapchain: Res, - resolution: Res, - environment_blend_mode: Res, + xr_frame_state: Option>, + views: Option>, + input: Option>, + swapchain: Option>, + resolution: Option>, + environment_blend_mode: Option>, // passthrough_layer: Option>, ) { + let xr_frame_state = xr_frame_state.unwrap(); + let views = views.unwrap(); + let input = input.unwrap(); + let swapchain = swapchain.unwrap(); + let resolution = resolution.unwrap(); + let environment_blend_mode = environment_blend_mode.unwrap(); { let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); diff --git a/src/resource_macros.rs b/src/resource_macros.rs index 933ce20..735ec50 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -1,7 +1,13 @@ #[macro_export] macro_rules! xr_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] + #[derive( + Clone, + bevy::prelude::Resource, + bevy::prelude::Deref, + bevy::prelude::DerefMut, + bevy::render::extract_resource::ExtractResource, + )] pub struct $wrapper_type($xr_type); impl $wrapper_type { @@ -29,7 +35,13 @@ macro_rules! xr_resource_wrapper { #[macro_export] macro_rules! xr_arc_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { - #[derive(Clone, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)] + #[derive( + Clone, + bevy::prelude::Resource, + bevy::prelude::Deref, + bevy::prelude::DerefMut, + bevy::render::extract_resource::ExtractResource, + )] pub struct $wrapper_type(std::sync::Arc<$xr_type>); impl $wrapper_type { diff --git a/src/resources.rs b/src/resources.rs index 62f2855..b73cb1c 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -2,11 +2,13 @@ use std::ffi::c_void; use std::sync::atomic::AtomicBool; use std::sync::Mutex; +use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; use crate::resource_macros::*; +use crate::xr_init::XrEnableStatus; use bevy::prelude::*; +use bevy::render::extract_resource::ExtractResourcePlugin; use openxr as xr; -use xr::ViewConfigurationView; xr_resource_wrapper!(XrInstance, xr::Instance); xr_resource_wrapper!(XrSession, xr::Session); @@ -32,6 +34,20 @@ pub(crate) enum OXrSessionSetupInfo { Vulkan(VulkanOXrSessionSetupInfo), } +pub struct XrResourcePlugin; + +impl Plugin for XrResourcePlugin { + fn build(&self, app: &mut App) { + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + } +} + pub enum Swapchain { Vulkan(SwapchainInner), } @@ -153,7 +169,7 @@ impl SwapchainInner { }, }; let swapchain = self.handle.lock().unwrap(); - if views.len() == 0 { + if views.is_empty() { warn!("views are len of 0"); return Ok(()); } diff --git a/src/xr_init.rs b/src/xr_init.rs index f60b0ba..698835d 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -11,16 +11,17 @@ use bevy::{ renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}, settings::WgpuSettings, }, - window::{RawHandleWrapper, PrimaryWindow}, + window::{PrimaryWindow, RawHandleWrapper}, }; use wgpu::Instance; use crate::{ + graphics, input::XrInput, resources::{ XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, - }, graphics, + }, }; #[derive(Resource, Clone)] @@ -52,14 +53,13 @@ pub enum XrEnableRequest { TryEnable, TryDisable, } -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq)] +#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Reflect, ExtractResource)] pub enum XrEnableStatus { Enabled, Disabled, - Waiting, } -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Debug)] +#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Debug, ExtractResource)] pub enum XrNextEnabledState { Enabled, Disabled, @@ -67,29 +67,33 @@ pub enum XrNextEnabledState { pub struct RenderRestartPlugin; -#[derive(Resource)] -pub struct ForceMain; - #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPreSetup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrSetup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPrePostSetup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostSetup; #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPreCleanup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrCleanup; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostCleanup; #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPreRenderUpdate; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrRenderUpdate; + #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostRenderUpdate; @@ -99,9 +103,9 @@ pub fn xr_only() -> impl FnMut(Option>) -> bool { impl Plugin for RenderRestartPlugin { fn build(&self, app: &mut App) { + info!("build RenderRestartPlugin"); add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()) - .insert_resource(ForceMain) .add_event::() .add_event::() .add_systems(PostStartup, setup_xr.run_if(xr_only())) @@ -174,13 +178,9 @@ pub fn update_xr_stuff(world: &mut World) { world.run_schedule(XrPostRenderUpdate); } -fn setup_xr_graphics() { +fn setup_xr_graphics() {} -} - -fn enable_xr( - -) {} +fn enable_xr() {} // fn handle_xr_enable_requests( // primary_window: Query<&RawHandleWrapper, With>, @@ -301,10 +301,10 @@ fn decide_next_xr_state( info!("Xr Already Disabled! ignoring request"); return; } - (_, Some(XrEnableStatus::Waiting)) => { - info!("Already Handling Request! ignoring request"); - return; - } + // (_, Some(XrEnableStatus::Waiting)) => { + // info!("Already Handling Request! ignoring request"); + // return; + // } _ => {} } let r = match request { diff --git a/src/xr_input/trackers.rs b/src/xr_input/trackers.rs index eea28c1..e607c16 100644 --- a/src/xr_input/trackers.rs +++ b/src/xr_input/trackers.rs @@ -1,3 +1,4 @@ +use bevy::hierarchy::Parent; use bevy::log::{debug, info}; use bevy::prelude::{ Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With, Without, @@ -30,17 +31,17 @@ pub struct OpenXRController; pub struct AimPose(pub Transform); pub fn adopt_open_xr_trackers( - query: Query>, + query: Query, Without)>, mut commands: Commands, - tracking_root_query: Query<(Entity, With)>, + tracking_root_query: Query>, ) { let root = tracking_root_query.get_single(); match root { - Ok(thing) => { + Ok(root) => { // info!("root is"); for tracker in query.iter() { info!("we got a new tracker"); - commands.entity(thing.0).add_child(tracker); + commands.entity(root).add_child(tracker); } } Err(_) => info!("root isnt spawned yet?"), From 144e786db940f99ca878e76965cd6539f23eda22 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Sun, 14 Jan 2024 02:12:49 +0100 Subject: [PATCH 03/25] ready to test --- src/lib.rs | 2 +- src/xr_init.rs | 79 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ea1e496..d1bd31a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -245,7 +245,7 @@ pub fn xr_begin_frame( session_running.store(false, std::sync::atomic::Ordering::Relaxed); } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { - app_exit.send(AppExit); + // app_exit.send(AppExit); return; } diff --git a/src/xr_init.rs b/src/xr_init.rs index 698835d..13ee973 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -8,7 +8,9 @@ use bevy::{ prelude::*, render::{ extract_resource::{ExtractResource, ExtractResourcePlugin}, - renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}, + renderer::{ + self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, + }, settings::WgpuSettings, }, window::{PrimaryWindow, RawHandleWrapper}, @@ -19,8 +21,8 @@ use crate::{ graphics, input::XrInput, resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, XrResolution, XrSession, - XrSessionRunning, XrSwapchain, XrViews, + OXrSessionSetupInfo, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, + XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }, }; @@ -97,17 +99,21 @@ pub struct XrRenderUpdate; #[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] pub struct XrPostRenderUpdate; -pub fn xr_only() -> impl FnMut(Option>) -> bool { - resource_exists_and_equals(XrEnableStatus::Enabled) +pub fn xr_only() -> impl FnMut(Option>, Res) -> bool { + |status, running| { + resource_exists_and_equals(XrEnableStatus::Enabled)(status) + && running.load(std::sync::atomic::Ordering::Relaxed) + } } impl Plugin for RenderRestartPlugin { fn build(&self, app: &mut App) { - info!("build RenderRestartPlugin"); add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()) .add_event::() .add_event::() + .add_event::() + .add_event::() .add_systems(PostStartup, setup_xr.run_if(xr_only())) .add_systems( PostUpdate, @@ -126,6 +132,8 @@ impl Plugin for RenderRestartPlugin { .chain(), ) .add_systems(XrCleanup, cleanup_oxr_session); + app.add_systems(PostUpdate, start_xr_session.run_if(on_event::())); + app.add_systems(PostUpdate, stop_xr_session.run_if(on_event::())); } } @@ -178,9 +186,64 @@ pub fn update_xr_stuff(world: &mut World) { world.run_schedule(XrPostRenderUpdate); } -fn setup_xr_graphics() {} +#[derive(Event, Clone, Copy, Default)] +pub struct StartXrSession; -fn enable_xr() {} +#[derive(Event, Clone, Copy, Default)] +pub struct EndXrSession; + +fn start_xr_session( + mut commands: Commands, + instance: Res, + primary_window: Query<&RawHandleWrapper, With>, + setup_info: NonSend, + render_device: Res, + render_adapter: Res, + render_instance: Res, +) { + let ( + xr_session, + xr_resolution, + xr_format, + xr_session_running, + xr_frame_waiter, + xr_swapchain, + xr_input, + xr_views, + xr_frame_state, + ) = match graphics::start_xr_session( + primary_window.get_single().cloned().ok(), + &setup_info, + &instance, + &render_device, + &render_adapter, + &render_instance, + ) { + Ok(data) => data, + Err(err) => { + error!("Unable to start OpenXR Session: {}", err); + return; + } + }; + commands.insert_resource(xr_session); + commands.insert_resource(xr_resolution); + commands.insert_resource(xr_format); + commands.insert_resource(xr_session_running); + commands.insert_resource(xr_frame_waiter); + commands.insert_resource(xr_swapchain); + commands.insert_resource(xr_input); + commands.insert_resource(xr_views); + commands.insert_resource(xr_frame_state); +} + +fn stop_xr_session(mut commands: Commands, session: ResMut) { + match session.request_exit() { + Ok(_) => {} + Err(err) => { + error!("Error while trying to request session exit: {}", err) + } + } +} // fn handle_xr_enable_requests( // primary_window: Query<&RawHandleWrapper, With>, From 87fe9093dfd14d4410d8cf5a0de038b407aac424 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 15 Jan 2024 05:08:49 +0100 Subject: [PATCH 04/25] fix android warning --- Cargo.toml | 4 ++++ src/lib.rs | 55 ++++++++++++++++++++++++++++++++++++------------ src/resources.rs | 8 +++++-- src/xr_init.rs | 21 +++++++++++++++--- 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2f6c6f5..bb93ce7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,10 @@ openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features "static", ] } +[target.'cfg(target_os = "android")'.dependencies] +ndk-context = "0.1" +jni = "0.20" + [dev-dependencies] bevy = "0.12" bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } diff --git a/src/lib.rs b/src/lib.rs index d1bd31a..4705e54 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod resources; pub mod xr_init; pub mod xr_input; +use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; use crate::xr_init::RenderRestartPlugin; @@ -70,6 +71,13 @@ pub struct FutureXrResources( impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { + app.insert_resource(XrSessionRunning::new(AtomicBool::new(false))); + // #[cfg(target_os = "android")] + // { + // let ctx = ndk_context::android_context(); + // let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); + // let env = vm.attach_current_thread_permanently(); + // } #[cfg(not(target_arch = "wasm32"))] match graphics::try_full_init( &mut app.world, @@ -91,8 +99,7 @@ impl Plugin for OpenXrPlugin { instance, ), }); - app.add_plugins(ExtractResourcePlugin::::default()); - app.insert_resource(XrEnableStatus::Enabled); + app.insert_resource(XrEnableStatus::Disabled); } Err(err) => { warn!("OpenXR Failed to initialize: {}", err); @@ -107,11 +114,17 @@ impl Plugin for OpenXrPlugin { app.add_plugins(RenderPlugin::default()); app.insert_resource(XrEnableStatus::Disabled); } + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_systems( + PreUpdate, + (xr_poll_events, xr_begin_frame.run_if(xr_only())).chain(), + ); } fn finish(&self, app: &mut App) { // TODO: Split this up into the indevidual resources - if app.world.get_resource::() == Some(&XrEnableStatus::Enabled) { + // app.world.get_resource::() == Some(&XrEnableStatus::Enabled) + if true { warn!("finished xr init"); let xr_instance = app .world @@ -152,7 +165,6 @@ impl Plugin for OpenXrPlugin { size: **xr_resolution, format: **xr_format, }; - app.add_systems(PreUpdate, xr_begin_frame.run_if(xr_only())); let mut manual_texture_views = app.world.resource_mut::(); manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); @@ -215,18 +227,13 @@ impl PluginGroup for DefaultXrPlugins { } } -pub fn xr_begin_frame( - instance: Res, - session: Res, +pub fn xr_poll_events( + instance: Option>, + session: Option>, session_running: Res, - mut frame_state: ResMut, - mut frame_waiter: ResMut, - swapchain: Res, - mut views: ResMut, - input: Res, mut app_exit: EventWriter, ) { - { + if let (Some(instance), Some(session)) = (instance, session) { let _span = info_span!("xr_poll_events"); while let Some(event) = instance.poll_event(&mut Default::default()).unwrap() { use xr::Event::*; @@ -237,6 +244,7 @@ pub fn xr_begin_frame( info!("entered XR state {:?}", e.state()); match e.state() { xr::SessionState::READY => { + info!("Calling Session begin :3"); session.begin(VIEW_TYPE).unwrap(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); } @@ -260,6 +268,16 @@ pub fn xr_begin_frame( } } } +} + +pub fn xr_begin_frame( + session: Res, + mut frame_state: ResMut, + mut frame_waiter: ResMut, + swapchain: Res, + mut views: ResMut, + input: Res, +) { { let _span = info_span!("xr_wait_frame").entered(); *frame_state = match frame_waiter.wait() { @@ -322,6 +340,8 @@ pub fn end_frame( swapchain: Option>, resolution: Option>, environment_blend_mode: Option>, + // _main_thread: NonSend<()>, + #[cfg(target_os = "android")] mut attached: Local, // passthrough_layer: Option>, ) { let xr_frame_state = xr_frame_state.unwrap(); @@ -330,6 +350,15 @@ pub fn end_frame( let swapchain = swapchain.unwrap(); let resolution = resolution.unwrap(); let environment_blend_mode = environment_blend_mode.unwrap(); + + #[cfg(target_os = "android")] + // if !*attached { + { + let ctx = ndk_context::android_context(); + let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); + let env = vm.attach_current_thread_as_daemon(); + *attached = true; + } { let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); diff --git a/src/resources.rs b/src/resources.rs index b73cb1c..584cb16 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -45,6 +45,7 @@ impl Plugin for XrResourcePlugin { app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); } } @@ -218,7 +219,8 @@ impl SwapchainInner { // } // None => - self.stream.lock().unwrap().end( + info!("swapchain stream lock"); + let r = self.stream.lock().unwrap().end( predicted_display_time, environment_blend_mode, &[&xr::CompositionLayerProjection::new().space(stage).views(&[ @@ -241,7 +243,9 @@ impl SwapchainInner { .image_rect(rect), ), ])], - ) + ); + info!("swapchain stream done"); + r // } } } diff --git a/src/xr_init.rs b/src/xr_init.rs index 13ee973..fb4917a 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -114,7 +114,14 @@ impl Plugin for RenderRestartPlugin { .add_event::() .add_event::() .add_event::() - .add_systems(PostStartup, setup_xr.run_if(xr_only())) + .add_systems( + PreUpdate, + setup_xr + .run_if(|running: Res| { + running.load(std::sync::atomic::Ordering::Relaxed) + }) + .run_if(run_once()), + ) .add_systems( PostUpdate, update_xr_stuff.run_if(on_event::()), @@ -132,8 +139,14 @@ impl Plugin for RenderRestartPlugin { .chain(), ) .add_systems(XrCleanup, cleanup_oxr_session); - app.add_systems(PostUpdate, start_xr_session.run_if(on_event::())); - app.add_systems(PostUpdate, stop_xr_session.run_if(on_event::())); + app.add_systems( + PostUpdate, + start_xr_session.run_if(on_event::()), + ); + app.add_systems( + PostUpdate, + stop_xr_session.run_if(on_event::()), + ); } } @@ -162,6 +175,8 @@ fn add_schedules(app: &mut App) { } pub fn setup_xr(world: &mut World) { + info!("running setup schedule :3"); + world.insert_resource(XrEnableStatus::Enabled); world.run_schedule(XrPreSetup); world.run_schedule(XrSetup); world.run_schedule(XrPrePostSetup); From 6f6948e6271010371594d1c2964e933a49fadfbe Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 23 Jan 2024 05:45:48 +0100 Subject: [PATCH 05/25] borked --- examples/demo/manifest.yaml | 1 + examples/demo/src/lib.rs | 28 +-- src/graphics/mod.rs | 2 +- src/lib.rs | 296 ++++++++++++------------ src/resources.rs | 6 +- src/xr_init.rs | 433 ------------------------------------ src/xr_input/hands/mod.rs | 46 +++- src/xr_input/mod.rs | 16 +- src/xr_input/xr_camera.rs | 66 ++++-- 9 files changed, 264 insertions(+), 630 deletions(-) delete mode 100644 src/xr_init.rs diff --git a/examples/demo/manifest.yaml b/examples/demo/manifest.yaml index 730187a..6cba5b8 100644 --- a/examples/demo/manifest.yaml +++ b/examples/demo/manifest.yaml @@ -13,6 +13,7 @@ android: # required: true uses_permission: - name: "com.oculus.permission.HAND_TRACKING" + - name: "android.permission.INTERNET" application: label: "Bevy Openxr Android" theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen" diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index 2867a23..e7feb94 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -19,7 +19,7 @@ use bevy_oxr::{ graphics::{extensions::XrExtensions, XrAppInfo, XrPreferdBlendMode}, input::XrInput, resources::{XrFrameState, XrInstance, XrSession}, - xr_init::{xr_only, XrEnableRequest, XrEnableStatus}, + xr_init::{xr_only, XrStatus}, xr_input::{ actions::XrActionSets, debug_gizmos::OpenXrDebugRenderer, @@ -41,18 +41,18 @@ use bevy_oxr::{ DefaultXrPlugins, }; -fn input_stuff( - keys: Res>, - status: Res, - mut request: EventWriter, -) { - if keys.just_pressed(KeyCode::Space) { - match status.into_inner() { - XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable), - XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable), - } - } -} +// fn input_stuff( +// keys: Res>, +// status: Res, +// mut request: EventWriter, +// ) { +// if keys.just_pressed(KeyCode::Space) { +// match status.into_inner() { +// XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable), +// XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable), +// } +// } +// } mod setup; use crate::setup::setup_scene; @@ -66,7 +66,7 @@ pub fn main() { let mut app = App::new(); let mut xr_extensions = XrExtensions::default(); - app.add_systems(Update, input_stuff) + app //lets get the usual diagnostic stuff added .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 24e80ff..33dc317 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -103,7 +103,7 @@ pub fn try_full_init( )> { let mut system_state: SystemState>> = SystemState::new(world); - let primary_window = system_state.get(&world).get_single().ok().cloned(); + let primary_window = system_state.get(world).get_single().ok().cloned(); let ( xr_instance, setup_info, diff --git a/src/lib.rs b/src/lib.rs index 4705e54..3b52be6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,10 +6,13 @@ pub mod resources; pub mod xr_init; pub mod xr_input; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::net::TcpStream; use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; -use crate::xr_init::RenderRestartPlugin; +use crate::xr_init::{StartXrSession, XrInitPlugin}; use crate::xr_input::hands::hand_tracking::DisableHandTracking; use crate::xr_input::oculus_touch::ActionSets; use bevy::app::{AppExit, PluginGroupBuilder}; @@ -18,22 +21,25 @@ use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; -use bevy::render::renderer::{render_system, RenderInstance}; +use bevy::render::renderer::RenderInstance; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; -use eyre::anyhow; use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; use input::XrInput; use openxr as xr; // use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; use resources::*; -use xr::{FormFactor, FrameWaiter}; -use xr_init::{xr_only, XrEnableStatus, XrRenderData}; +use xr::FormFactor; +use xr_init::{ + xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, +}; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin}; +use xr_input::hands::XrHandPlugins; +use xr_input::xr_camera::XrCameraType; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -49,141 +55,119 @@ pub struct OpenXrPlugin { app_info: XrAppInfo, } -#[derive(Resource)] -pub struct FutureXrResources( - pub Arc< - Mutex< - Option<( - XrInstance, - XrSession, - XrEnvironmentBlendMode, - XrResolution, - XrFormat, - XrSessionRunning, - XrSwapchain, - XrInput, - XrViews, - XrFrameState, - )>, - >, - >, -); - impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { app.insert_resource(XrSessionRunning::new(AtomicBool::new(false))); - // #[cfg(target_os = "android")] - // { - // let ctx = ndk_context::android_context(); - // let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); - // let env = vm.attach_current_thread_permanently(); - // } #[cfg(not(target_arch = "wasm32"))] - match graphics::try_full_init( - &mut app.world, + match graphics::initialize_xr_instance( + SystemState::>>::new(&mut app.world) + .get(&app.world) + .get_single() + .ok() + .cloned(), self.reqeusted_extensions.clone(), self.prefered_blend_mode, self.app_info.clone(), ) { - Ok((device, queue, adapter_info, render_adapter, instance)) => { + Ok(( + xr_instance, + oxr_session_setup_info, + blend_mode, + device, + queue, + adapter_info, + render_adapter, + instance, + )) => { debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); debug!("Configured wgpu adapter Features: {:#?}", device.features()); - warn!("Starting in Xr"); + warn!("Starting with OpenXR Instance"); app.insert_resource(ActionSets(vec![])); + app.insert_resource(xr_instance); + app.insert_resource(blend_mode); + app.insert_non_send_resource(oxr_session_setup_info); + let render_instance = RenderInstance(instance.into()); + app.insert_resource(render_instance.clone()); app.add_plugins(RenderPlugin { render_creation: RenderCreation::Manual( device, queue, adapter_info, render_adapter, - instance, + render_instance, ), }); - app.insert_resource(XrEnableStatus::Disabled); + app.insert_resource(XrStatus::Disabled); + app.world.send_event(StartXrSession); } Err(err) => { - warn!("OpenXR Failed to initialize: {}", err); + warn!("OpenXR Instance Failed to initialize: {}", err); app.add_plugins(RenderPlugin::default()); - app.add_plugins(ExtractResourcePlugin::::default()); - app.insert_resource(XrEnableStatus::Disabled); + app.insert_resource(XrStatus::NoInstance); } } - // app.add_systems(PreUpdate, mr_test); #[cfg(target_arch = "wasm32")] { app.add_plugins(RenderPlugin::default()); - app.insert_resource(XrEnableStatus::Disabled); + app.insert_resource(XrStatus::Disabled); } - app.add_plugins(ExtractResourcePlugin::::default()); app.add_systems( PreUpdate, - (xr_poll_events, xr_begin_frame.run_if(xr_only())).chain(), + xr_poll_events.run_if(|status: Res| *status != XrStatus::NoInstance), ); - } - - fn finish(&self, app: &mut App) { - // TODO: Split this up into the indevidual resources - // app.world.get_resource::() == Some(&XrEnableStatus::Enabled) - if true { - warn!("finished xr init"); - let xr_instance = app - .world - .get_resource::() - .expect("should exist"); - let xr_session = app.world.get_resource::().expect("should exist"); - let hands = xr_instance.exts().ext_hand_tracking.is_some() - && xr_instance - .supports_hand_tracking( - xr_instance - .system(FormFactor::HEAD_MOUNTED_DISPLAY) - .unwrap(), - ) - .is_ok_and(|v| v); - if hands { - app.insert_resource(HandTrackingData::new(xr_session).unwrap()); - } else { - app.insert_resource(DisableHandTracking::Both); - } - let xr_swapchain = app - .world - .get_resource::() - .expect("should exist"); - let xr_resolution = app - .world - .get_resource::() - .expect("should exist"); - let xr_format = app.world.get_resource::().expect("should exist"); - - let (left, right) = xr_swapchain.get_render_views(); - let left = ManualTextureView { - texture_view: left.into(), - size: **xr_resolution, - format: **xr_format, - }; - let right = ManualTextureView { - texture_view: right.into(), - size: **xr_resolution, - format: **xr_format, - }; - let mut manual_texture_views = app.world.resource_mut::(); - manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); - manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); - drop(manual_texture_views); - let render_app = app.sub_app_mut(RenderApp); - render_app.add_systems( - Render, - ( - post_frame - .run_if(xr_only()) - .before(render_system) - .after(RenderSet::ExtractCommands), - end_frame.run_if(xr_only()).after(render_system), - ), - ); - } + app.add_systems( + PreUpdate, + ( + xr_reset_should_render, + apply_deferred, + xr_wait_frame.run_if(xr_only()), + apply_deferred, + locate_views.run_if(xr_only()), + apply_deferred, + ) + .chain() + .after(xr_poll_events), + ); + let render_app = app.sub_app_mut(RenderApp); + render_app.add_systems( + Render, + xr_begin_frame + .run_if(xr_only()) + .run_if(xr_render_only()) + .after(RenderSet::ExtractCommands) + .before(xr_pre_frame), + ); + render_app.add_systems( + Render, + xr_pre_frame + .run_if(xr_only()) + .run_if(xr_render_only()) + .in_set(RenderSet::Prepare), + ); + render_app.add_systems( + Render, + (locate_views, xr_input::xr_camera::xr_camera_head_sync) + .chain() + .run_if(xr_only()) + .run_if(xr_render_only()) + .in_set(RenderSet::Prepare), + ); + render_app.add_systems( + Render, + xr_end_frame + .run_if(xr_only()) + .run_if(xr_render_only()) + .after(RenderSet::Render), + ); + render_app.insert_resource(TcpConnection( + TcpStream::connect("192.168.2.100:6969").unwrap(), + )); } } +#[derive(Resource)] +struct TcpConnection(TcpStream); + #[derive(Default)] pub struct DefaultXrPlugins { pub reqeusted_extensions: XrExtensions, @@ -203,11 +187,10 @@ impl PluginGroup for DefaultXrPlugins { app_info: self.app_info.clone(), }) .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) - .add_before::(RenderRestartPlugin) - .add(HandEmulationPlugin) - .add(HandTrackingPlugin) + .add_after::(XrInitPlugin) + .add_before::(XrEarlyInitPlugin) + .add(XrHandPlugins) .add(XrResourcePlugin) - // .add(xr_init::RenderRestartPlugin) .set(WindowPlugin { #[cfg(not(target_os = "android"))] primary_window: Some(Window { @@ -227,11 +210,16 @@ impl PluginGroup for DefaultXrPlugins { } } -pub fn xr_poll_events( +fn xr_reset_should_render(mut should: ResMut) { + **should = false; +} + +fn xr_poll_events( instance: Option>, session: Option>, session_running: Res, mut app_exit: EventWriter, + mut cleanup_xr: EventWriter, ) { if let (Some(instance), Some(session)) = (instance, session) { let _span = info_span!("xr_poll_events"); @@ -251,6 +239,7 @@ pub fn xr_poll_events( xr::SessionState::STOPPING => { session.end().unwrap(); session_running.store(false, std::sync::atomic::Ordering::Relaxed); + cleanup_xr.send_default(); } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { // app_exit.send(AppExit); @@ -260,7 +249,9 @@ pub fn xr_poll_events( _ => {} } } - InstanceLossPending(_) => return, + InstanceLossPending(_) => { + app_exit.send_default(); + } EventsLost(e) => { warn!("lost {} XR events", e.lost_event_count()); } @@ -270,13 +261,15 @@ pub fn xr_poll_events( } } -pub fn xr_begin_frame( - session: Res, +fn xr_begin_frame(swapchain: Res) { + let _span = info_span!("xr_begin_frame").entered(); + swapchain.begin().unwrap() +} + +pub fn xr_wait_frame( mut frame_state: ResMut, mut frame_waiter: ResMut, - swapchain: Res, - mut views: ResMut, - input: Res, + mut should_render: ResMut, ) { { let _span = info_span!("xr_wait_frame").entered(); @@ -287,21 +280,11 @@ pub fn xr_begin_frame( return; } }; - } - { - let _span = info_span!("xr_begin_frame").entered(); - swapchain.begin().unwrap() - } - { - let _span = info_span!("xr_locate_views").entered(); - **views = session - .locate_views(VIEW_TYPE, frame_state.predicted_display_time, &input.stage) - .unwrap() - .1; + **should_render = frame_state.should_render; } } -pub fn post_frame( +pub fn xr_pre_frame( resolution: Res, format: Res, swapchain: Res, @@ -313,7 +296,9 @@ pub fn post_frame( } { let _span = info_span!("xr_wait_image").entered(); + info!("wait image"); swapchain.wait_image().unwrap(); + info!("waited image"); } { let _span = info_span!("xr_update_manual_texture_views").entered(); @@ -333,36 +318,51 @@ pub fn post_frame( } } -pub fn end_frame( - xr_frame_state: Option>, - views: Option>, - input: Option>, - swapchain: Option>, - resolution: Option>, - environment_blend_mode: Option>, - // _main_thread: NonSend<()>, - #[cfg(target_os = "android")] mut attached: Local, - // passthrough_layer: Option>, +pub fn xr_end_frame( + xr_frame_state: Res, + views: Res, + input: Res, + swapchain: Res, + resolution: Res, + environment_blend_mode: Res, + mut connection: ResMut, + cams: Query<(&Transform, &XrCameraType)>, ) { - let xr_frame_state = xr_frame_state.unwrap(); - let views = views.unwrap(); - let input = input.unwrap(); - let swapchain = swapchain.unwrap(); - let resolution = resolution.unwrap(); - let environment_blend_mode = environment_blend_mode.unwrap(); - #[cfg(target_os = "android")] - // if !*attached { { let ctx = ndk_context::android_context(); let vm = unsafe { jni::JavaVM::from_raw(ctx.vm().cast()) }.unwrap(); let env = vm.attach_current_thread_as_daemon(); - *attached = true; } { let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); } + let mut cam = None; + for (t, c) in &cams { + if *c == XrCameraType::Xr(xr_input::xr_camera::Eye::Left) { + cam = Some(*t); + break; + } + } + let _ = std::writeln!( + &mut connection.0, + "{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + views[0].pose.position.x, + views[0].pose.position.y, + views[0].pose.position.z, + views[0].pose.orientation.x, + views[0].pose.orientation.y, + views[0].pose.orientation.z, + views[0].pose.orientation.w, + cam.unwrap().translation.x, + cam.unwrap().translation.y, + cam.unwrap().translation.z, + cam.unwrap().rotation.x, + cam.unwrap().rotation.y, + cam.unwrap().rotation.z, + cam.unwrap().rotation.w, + ); { let _span = info_span!("xr_end_frame").entered(); let result = swapchain.end( diff --git a/src/resources.rs b/src/resources.rs index 584cb16..da72cf6 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -5,7 +5,7 @@ use std::sync::Mutex; use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; use crate::resource_macros::*; -use crate::xr_init::XrEnableStatus; +use crate::xr_init::XrStatus; use bevy::prelude::*; use bevy::render::extract_resource::ExtractResourcePlugin; use openxr as xr; @@ -27,7 +27,6 @@ pub(crate) struct VulkanOXrSessionSetupInfo { pub(crate) vk_instance_ptr: *const c_void, pub(crate) queue_family_index: u32, pub(crate) xr_system_id: xr::SystemId, - // pub(crate) swapchain_create_info: xr::SwapchainCreateInfo } pub(crate) enum OXrSessionSetupInfo { @@ -46,6 +45,7 @@ impl Plugin for XrResourcePlugin { app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); } } @@ -219,7 +219,6 @@ impl SwapchainInner { // } // None => - info!("swapchain stream lock"); let r = self.stream.lock().unwrap().end( predicted_display_time, environment_blend_mode, @@ -244,7 +243,6 @@ impl SwapchainInner { ), ])], ); - info!("swapchain stream done"); r // } } diff --git a/src/xr_init.rs b/src/xr_init.rs deleted file mode 100644 index fb4917a..0000000 --- a/src/xr_init.rs +++ /dev/null @@ -1,433 +0,0 @@ -// Just a lot of code that is meant for something way more complex but hey. -// maybe will work on that soon - -use std::sync::Arc; - -use bevy::{ - ecs::schedule::{ExecutorKind, ScheduleLabel}, - prelude::*, - render::{ - extract_resource::{ExtractResource, ExtractResourcePlugin}, - renderer::{ - self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue, - }, - settings::WgpuSettings, - }, - window::{PrimaryWindow, RawHandleWrapper}, -}; -use wgpu::Instance; - -use crate::{ - graphics, - input::XrInput, - resources::{ - OXrSessionSetupInfo, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrInstance, - XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, - }, -}; - -#[derive(Resource, Clone)] -pub struct RenderCreationData { - pub device: RenderDevice, - pub queue: RenderQueue, - pub adapter_info: RenderAdapterInfo, - pub render_adapter: RenderAdapter, - pub instance: Arc, -} - -#[derive(Resource, Clone, ExtractResource)] -pub struct XrRenderData { - pub xr_instance: XrInstance, - pub xr_session: XrSession, - pub xr_blend_mode: XrEnvironmentBlendMode, - pub xr_resolution: XrResolution, - pub xr_format: XrFormat, - pub xr_session_running: XrSessionRunning, - // pub xr_frame_waiter: XrFrameWaiter, - pub xr_swapchain: XrSwapchain, - pub xr_input: XrInput, - pub xr_views: XrViews, - pub xr_frame_state: XrFrameState, -} - -#[derive(Event, Clone, Copy, Debug)] -pub enum XrEnableRequest { - TryEnable, - TryDisable, -} -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Reflect, ExtractResource)] -pub enum XrEnableStatus { - Enabled, - Disabled, -} - -#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Debug, ExtractResource)] -pub enum XrNextEnabledState { - Enabled, - Disabled, -} - -pub struct RenderRestartPlugin; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPreSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPrePostSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPostSetup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPreCleanup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrCleanup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPostCleanup; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPreRenderUpdate; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrRenderUpdate; - -#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] -pub struct XrPostRenderUpdate; - -pub fn xr_only() -> impl FnMut(Option>, Res) -> bool { - |status, running| { - resource_exists_and_equals(XrEnableStatus::Enabled)(status) - && running.load(std::sync::atomic::Ordering::Relaxed) - } -} - -impl Plugin for RenderRestartPlugin { - fn build(&self, app: &mut App) { - add_schedules(app); - app.add_plugins(ExtractResourcePlugin::::default()) - .add_event::() - .add_event::() - .add_event::() - .add_event::() - .add_systems( - PreUpdate, - setup_xr - .run_if(|running: Res| { - running.load(std::sync::atomic::Ordering::Relaxed) - }) - .run_if(run_once()), - ) - .add_systems( - PostUpdate, - update_xr_stuff.run_if(on_event::()), - ) - .add_systems(XrPreRenderUpdate, decide_next_xr_state) - .add_systems(XrPostRenderUpdate, clear_events) - .add_systems( - XrRenderUpdate, - ( - cleanup_xr.run_if(resource_exists_and_equals(XrNextEnabledState::Disabled)), - // handle_xr_enable_requests, - apply_deferred, - setup_xr, /* .run_if(resource_exists_and_equals(XrEnableStatus::Enabled)) */ - ) - .chain(), - ) - .add_systems(XrCleanup, cleanup_oxr_session); - app.add_systems( - PostUpdate, - start_xr_session.run_if(on_event::()), - ); - app.add_systems( - PostUpdate, - stop_xr_session.run_if(on_event::()), - ); - } -} - -fn clear_events(mut events: ResMut>) { - events.clear(); -} - -fn add_schedules(app: &mut App) { - let schedules = [ - Schedule::new(XrPreSetup), - Schedule::new(XrSetup), - Schedule::new(XrPrePostSetup), - Schedule::new(XrPostSetup), - Schedule::new(XrPreRenderUpdate), - Schedule::new(XrRenderUpdate), - Schedule::new(XrPostRenderUpdate), - Schedule::new(XrPreCleanup), - Schedule::new(XrCleanup), - Schedule::new(XrPostCleanup), - ]; - for mut schedule in schedules { - schedule.set_executor_kind(ExecutorKind::SingleThreaded); - schedule.set_apply_final_deferred(true); - app.add_schedule(schedule); - } -} - -pub fn setup_xr(world: &mut World) { - info!("running setup schedule :3"); - world.insert_resource(XrEnableStatus::Enabled); - world.run_schedule(XrPreSetup); - world.run_schedule(XrSetup); - world.run_schedule(XrPrePostSetup); - world.run_schedule(XrPostSetup); -} -fn cleanup_xr(world: &mut World) { - world.run_schedule(XrPreCleanup); - world.run_schedule(XrCleanup); - world.run_schedule(XrPostCleanup); -} - -fn cleanup_oxr_session(xr_status: Option>, session: Option>) { - if let (Some(XrEnableStatus::Disabled), Some(s)) = (xr_status.map(|v| v.into_inner()), session) - { - s.into_inner().request_exit().unwrap(); - } -} - -pub fn update_xr_stuff(world: &mut World) { - world.run_schedule(XrPreRenderUpdate); - world.run_schedule(XrRenderUpdate); - world.run_schedule(XrPostRenderUpdate); -} - -#[derive(Event, Clone, Copy, Default)] -pub struct StartXrSession; - -#[derive(Event, Clone, Copy, Default)] -pub struct EndXrSession; - -fn start_xr_session( - mut commands: Commands, - instance: Res, - primary_window: Query<&RawHandleWrapper, With>, - setup_info: NonSend, - render_device: Res, - render_adapter: Res, - render_instance: Res, -) { - let ( - xr_session, - xr_resolution, - xr_format, - xr_session_running, - xr_frame_waiter, - xr_swapchain, - xr_input, - xr_views, - xr_frame_state, - ) = match graphics::start_xr_session( - primary_window.get_single().cloned().ok(), - &setup_info, - &instance, - &render_device, - &render_adapter, - &render_instance, - ) { - Ok(data) => data, - Err(err) => { - error!("Unable to start OpenXR Session: {}", err); - return; - } - }; - commands.insert_resource(xr_session); - commands.insert_resource(xr_resolution); - commands.insert_resource(xr_format); - commands.insert_resource(xr_session_running); - commands.insert_resource(xr_frame_waiter); - commands.insert_resource(xr_swapchain); - commands.insert_resource(xr_input); - commands.insert_resource(xr_views); - commands.insert_resource(xr_frame_state); -} - -fn stop_xr_session(mut commands: Commands, session: ResMut) { - match session.request_exit() { - Ok(_) => {} - Err(err) => { - error!("Error while trying to request session exit: {}", err) - } - } -} - -// fn handle_xr_enable_requests( -// primary_window: Query<&RawHandleWrapper, With>, -// mut commands: Commands, -// next_state: Res, -// on_main: Option>, -// ) { -// // Just to force this system onto the main thread because of unsafe code -// let _ = on_main; -// -// let (creation_data, xr_data) = match next_state.into_inner() { -// XrNextEnabledState::Enabled => { -// let ( -// device, -// queue, -// adapter_info, -// render_adapter, -// instance, -// xr_instance, -// session, -// blend_mode, -// resolution, -// format, -// session_running, -// frame_waiter, -// swapchain, -// input, -// views, -// frame_state, -// ) = graphics::initialize_xr_graphics(primary_window.get_single().ok().cloned()) -// .unwrap(); -// -// commands.insert_resource(XrEnableStatus::Enabled); -// ( -// RenderCreationData { -// device, -// queue, -// adapter_info, -// render_adapter, -// instance: Arc::new(instance), -// }, -// Some(XrRenderData { -// xr_instance, -// xr_session: session, -// xr_blend_mode: blend_mode, -// xr_resolution: resolution, -// xr_format: format, -// xr_session_running: session_running, -// xr_frame_waiter: frame_waiter, -// xr_swapchain: swapchain, -// xr_input: input, -// xr_views: views, -// xr_frame_state: frame_state, -// }), -// ) -// } -// XrNextEnabledState::Disabled => ( -// init_non_xr_graphics(primary_window.get_single().ok().cloned()), -// None, -// ), -// }; -// -// commands.insert_resource(creation_data.device); -// commands.insert_resource(creation_data.queue); -// commands.insert_resource(RenderInstance(creation_data.instance)); -// commands.insert_resource(creation_data.adapter_info); -// commands.insert_resource(creation_data.render_adapter); -// -// if let Some(xr_data) = xr_data { -// // TODO: Remove this lib.rs:144 -// commands.insert_resource(xr_data.clone()); -// -// commands.insert_resource(xr_data.xr_instance); -// commands.insert_resource(xr_data.xr_session); -// commands.insert_resource(xr_data.xr_blend_mode); -// commands.insert_resource(xr_data.xr_resolution); -// commands.insert_resource(xr_data.xr_format); -// commands.insert_resource(xr_data.xr_session_running); -// commands.insert_resource(xr_data.xr_frame_waiter); -// commands.insert_resource(xr_data.xr_input); -// commands.insert_resource(xr_data.xr_views); -// commands.insert_resource(xr_data.xr_frame_state); -// commands.insert_resource(xr_data.xr_swapchain); -// } else { -// commands.remove_resource::(); -// -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// commands.remove_resource::(); -// } -// } - -fn decide_next_xr_state( - mut commands: Commands, - mut events: EventReader, - xr_status: Option>, -) { - info!("hm"); - let request = match events.read().next() { - Some(v) => v, - None => return, - }; - info!("ok"); - match (request, xr_status.as_deref()) { - (XrEnableRequest::TryEnable, Some(XrEnableStatus::Enabled)) => { - info!("Xr Already Enabled! ignoring request"); - return; - } - (XrEnableRequest::TryDisable, Some(XrEnableStatus::Disabled)) => { - info!("Xr Already Disabled! ignoring request"); - return; - } - // (_, Some(XrEnableStatus::Waiting)) => { - // info!("Already Handling Request! ignoring request"); - // return; - // } - _ => {} - } - let r = match request { - XrEnableRequest::TryEnable => XrNextEnabledState::Enabled, - XrEnableRequest::TryDisable => XrNextEnabledState::Disabled, - }; - info!("{:#?}", r); - commands.insert_resource(r); -} - -pub fn init_non_xr_graphics(primary_window: Option) -> RenderCreationData { - let settings = WgpuSettings::default(); - - let async_renderer = async move { - let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { - // Probably a bad idea unwraping here but on the other hand no backends - backends: settings.backends.unwrap(), - dx12_shader_compiler: settings.dx12_shader_compiler.clone(), - }); - let surface = primary_window.map(|wrapper| unsafe { - // SAFETY: Plugins should be set up on the main thread. - let handle = wrapper.get_handle(); - instance - .create_surface(&handle) - .expect("Failed to create wgpu surface") - }); - - let request_adapter_options = wgpu::RequestAdapterOptions { - power_preference: settings.power_preference, - compatible_surface: surface.as_ref(), - ..Default::default() - }; - - let (device, queue, adapter_info, render_adapter) = - renderer::initialize_renderer(&instance, &settings, &request_adapter_options).await; - debug!("Configured wgpu adapter Limits: {:#?}", device.limits()); - debug!("Configured wgpu adapter Features: {:#?}", device.features()); - RenderCreationData { - device, - queue, - adapter_info, - render_adapter, - instance: Arc::new(instance), - } - }; - // No need for wasm in bevy_oxr web xr would be a different crate - futures_lite::future::block_on(async_renderer) -} diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs index 06eb5c1..fa85cf8 100644 --- a/src/xr_input/hands/mod.rs +++ b/src/xr_input/hands/mod.rs @@ -1,6 +1,15 @@ use bevy::{app::PluginGroupBuilder, prelude::*}; +use openxr::FormFactor; -use self::{emulated::HandEmulationPlugin, hand_tracking::HandTrackingPlugin}; +use crate::{ + resources::{XrInstance, XrSession}, + xr_init::XrPreSetup, +}; + +use self::{ + emulated::HandEmulationPlugin, + hand_tracking::{DisableHandTracking, HandTrackingData, HandTrackingPlugin}, +}; pub mod common; pub mod emulated; @@ -8,12 +17,35 @@ pub mod hand_tracking; pub struct XrHandPlugins; -impl PluginGroup for XrHandPlugins { - fn build(self) -> PluginGroupBuilder { - PluginGroupBuilder::start::() - .add(HandTrackingPlugin) - .add(HandEmulationPlugin) - .build() +impl Plugin for XrHandPlugins { + fn build(&self, app: &mut App) { + app.add_plugins(HandTrackingPlugin) + .add_plugins(HandPlugin) + .add_plugins(HandEmulationPlugin); + } +} + +pub struct HandPlugin; + +impl Plugin for HandPlugin { + fn build(&self, app: &mut App) { + app.add_systems(XrPreSetup, check_for_handtracking); + } +} + +fn check_for_handtracking( + mut commands: Commands, + instance: Res, + session: Res, +) { + let hands = instance.exts().ext_hand_tracking.is_some() + && instance + .supports_hand_tracking(instance.system(FormFactor::HEAD_MOUNTED_DISPLAY).unwrap()) + .is_ok_and(|v| v); + if hands { + commands.insert_resource(HandTrackingData::new(&session).unwrap()); + } else { + commands.insert_resource(DisableHandTracking::Both); } } diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index 540f2e2..ac2e13b 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -10,11 +10,11 @@ pub mod trackers; pub mod xr_camera; use crate::resources::{XrInstance, XrSession}; -use crate::xr_begin_frame; -use crate::xr_init::{xr_only, XrPostSetup, XrSetup, XrPreSetup}; +use crate::xr_init::{xr_only, XrPostSetup, XrPreSetup, XrSetup}; use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; +use crate::{locate_views, xr_wait_frame}; use bevy::app::{App, PostUpdate, Startup}; use bevy::ecs::entity::Entity; use bevy::ecs::query::With; @@ -24,17 +24,19 @@ use bevy::math::Vec2; use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource}; use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3}; use bevy::render::camera::CameraProjectionPlugin; +use bevy::render::extract_component::ExtractComponentPlugin; use bevy::render::view::{update_frusta, VisibilitySystems}; use bevy::transform::TransformSystem; use bevy::utils::HashMap; use openxr::Binding; use self::actions::{setup_oxr_actions, OpenXrActionsPlugin}; -use self::oculus_touch::{post_action_setup_oculus_controller, ActionSets, init_subaction_path}; +use self::oculus_touch::{init_subaction_path, post_action_setup_oculus_controller, ActionSets}; use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, }; +use self::xr_camera::{XrCameraType, TransformExtract}; #[derive(Copy, Clone)] pub struct OpenXrInput { @@ -67,7 +69,10 @@ impl Plugin for OpenXrInput { app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); app.add_systems( PreUpdate, - xr_camera_head_sync.run_if(xr_only()).after(xr_begin_frame), + xr_camera_head_sync + .run_if(xr_only()) + .after(xr_wait_frame) + .after(locate_views), ); //update controller trackers app.add_systems(Update, update_open_xr_controllers.run_if(xr_only())); @@ -79,6 +84,9 @@ impl Plugin for OpenXrInput { ); app.add_systems(XrPreSetup, init_subaction_path); app.add_systems(XrSetup, setup_xr_cameras); + app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); } } diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 210feaa..1d143c3 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -1,9 +1,11 @@ use crate::xr_input::{QuatConv, Vec3Conv}; use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; +use bevy::ecs::system::lifetimeless::Read; use bevy::math::Vec3A; use bevy::prelude::*; use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; +use bevy::render::extract_component::ExtractComponent; use bevy::render::primitives::Frustum; use bevy::render::view::{ColorGrading, VisibleEntities}; use openxr::Fovf; @@ -48,6 +50,35 @@ pub enum XrCameraType { Flatscreen, } + +#[derive(Component)] +pub(super) struct TransformExtract; + + +impl ExtractComponent for TransformExtract { + type Query = Read; + + type Filter = (); + + type Out = Transform; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + Some(*item) + } +} + +impl ExtractComponent for XrCameraType { + type Query = Read; + + type Filter = (); + + type Out = Self; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + Some(*item) + } +} + #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, @@ -81,7 +112,7 @@ impl XrCameraBundle { } } -#[derive(Debug, Clone, Component, Reflect)] +#[derive(Debug, Clone, Component, Reflect, ExtractComponent)] #[reflect(Component, Default)] pub struct XRProjection { pub near: f32, @@ -241,24 +272,21 @@ impl CameraProjection for XRProjection { pub fn xr_camera_head_sync( views: ResMut, - query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, ) { - fn f( - views: ResMut, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, - ) -> Option<()> { - //TODO calculate HMD position - for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - XrCameraType::Xr(eye) => *eye as usize, - XrCameraType::Flatscreen => return None, - }; - let view = views.get(view_idx)?; - xr_projection.fov = view.fov; - transform.rotation = view.pose.orientation.to_quat(); - transform.translation = view.pose.position.to_vec3(); - } - Some(()) + //TODO calculate HMD position + for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { + let view_idx = match camera_type { + XrCameraType::Xr(eye) => *eye as usize, + // I don't belive we need a flatscrenn cam, that's just a cam without this component + XrCameraType::Flatscreen => continue, + }; + let view = match views.get(view_idx) { + Some(views) => views, + None => continue, + }; + xr_projection.fov = view.fov; + transform.rotation = view.pose.orientation.to_quat(); + transform.translation = view.pose.position.to_vec3(); } - let _ = f(views, query); } From a0ac82b21c3e79b46bb27eded4a560f888bf8e9d Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 23 Jan 2024 05:46:01 +0100 Subject: [PATCH 06/25] borked --- + | 294 +++++++++++++++++++++++++++++++++++++++ src/xr_init/mod.rs | 197 ++++++++++++++++++++++++++ src/xr_init/schedules.rs | 51 +++++++ 3 files changed, 542 insertions(+) create mode 100644 + create mode 100644 src/xr_init/mod.rs create mode 100644 src/xr_init/schedules.rs diff --git a/+ b/+ new file mode 100644 index 0000000..8e33f19 --- /dev/null +++ b/+ @@ -0,0 +1,294 @@ +use crate::xr_input::{QuatConv, Vec3Conv}; +use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; +use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; +use bevy::ecs::system::lifetimeless::Read; +use bevy::math::Vec3A; +use bevy::prelude::*; +use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; +use bevy::render::extract_component::ExtractComponent; +use bevy::render::primitives::Frustum; +use bevy::render::view::{ColorGrading, VisibleEntities}; +use openxr::Fovf; + +#[derive(Bundle)] +pub struct XrCamerasBundle { + pub left: XrCameraBundle, + pub right: XrCameraBundle, +} +impl XrCamerasBundle { + pub fn new() -> Self { + Self::default() + } +} +impl Default for XrCamerasBundle { + fn default() -> Self { + Self { + left: XrCameraBundle::new(Eye::Left), + right: XrCameraBundle::new(Eye::Right), + } + } +} + +#[derive(Bundle)] +pub struct XrCameraBundle { + pub camera: Camera, + pub camera_render_graph: CameraRenderGraph, + pub xr_projection: XRProjection, + pub visible_entities: VisibleEntities, + pub frustum: Frustum, + pub transform: Transform, + pub global_transform: GlobalTransform, + pub camera_3d: Camera3d, + pub tonemapping: Tonemapping, + pub dither: DebandDither, + pub color_grading: ColorGrading, + pub xr_camera_type: XrCameraType, +} +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] +pub enum XrCameraType { + Xr(Eye), + Flatscreen, +} + + +#[derive(Component)] +pub(super) struct TransformExtract; + + +impl ExtractComponent for TransformExtract { + type Query = Read; + + type Filter = (); + + type Out = Transform; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + info!("extracting Transform"); + Some(*item) + } +} + +impl ExtractComponent for XrCameraType { + type Query = Read; + + type Filter = (); + + type Out = Self; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + info!("extracting Cam Type"); + Some(*item) + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub enum Eye { + Left = 0, + Right = 1, +} + +impl XrCameraBundle { + pub fn new(eye: Eye) -> Self { + Self { + camera: Camera { + order: -1, + target: RenderTarget::TextureView(match eye { + Eye::Left => LEFT_XR_TEXTURE_HANDLE, + Eye::Right => RIGHT_XR_TEXTURE_HANDLE, + }), + viewport: None, + ..default() + }, + camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME), + xr_projection: Default::default(), + visible_entities: Default::default(), + frustum: Default::default(), + transform: Default::default(), + global_transform: Default::default(), + camera_3d: Default::default(), + tonemapping: Default::default(), + dither: DebandDither::Enabled, + color_grading: Default::default(), + xr_camera_type: XrCameraType::Xr(eye), + } + } +} + +#[derive(Debug, Clone, Component, Reflect, ExtractComponent)] +#[reflect(Component, Default)] +pub struct XRProjection { + pub near: f32, + pub far: f32, + #[reflect(ignore)] + pub fov: Fovf, +} + +impl Default for XRProjection { + fn default() -> Self { + Self { + near: 0.1, + far: 1000., + fov: Default::default(), + } + } +} + +impl XRProjection { + pub fn new(near: f32, far: f32, fov: Fovf) -> Self { + XRProjection { near, far, fov } + } +} + +impl CameraProjection for XRProjection { + // ============================================================================= + // math code adapted from + // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h + // Copyright (c) 2017 The Khronos Group Inc. + // Copyright (c) 2016 Oculus VR, LLC. + // SPDX-License-Identifier: Apache-2.0 + // ============================================================================= + fn get_projection_matrix(&self) -> Mat4 { + // symmetric perspective for debugging + // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); + // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); + // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); + + let fov = self.fov; + let is_vulkan_api = false; // FIXME wgpu probably abstracts this + let near_z = self.near; + let far_z = -1.; // use infinite proj + // let far_z = self.far; + + let tan_angle_left = fov.angle_left.tan(); + let tan_angle_right = fov.angle_right.tan(); + + let tan_angle_down = fov.angle_down.tan(); + let tan_angle_up = fov.angle_up.tan(); + + let tan_angle_width = tan_angle_right - tan_angle_left; + + // Set to tanAngleDown - tanAngleUp for a clip space with positive Y + // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with + // positive Y up (OpenGL / D3D / Metal). + // const float tanAngleHeight = + // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); + let tan_angle_height = if is_vulkan_api { + tan_angle_down - tan_angle_up + } else { + tan_angle_up - tan_angle_down + }; + + // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). + // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). + // const float offsetZ = + // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; + // FIXME handle enum of graphics apis + let offset_z = 0.; + + let mut cols: [f32; 16] = [0.0; 16]; + + if far_z <= near_z { + // place the far plane at infinity + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -1.; + cols[14] = -(near_z + offset_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + + // bevy uses the _reverse_ infinite projection + // https://dev.theomader.com/depth-precision/ + let z_reversal = Mat4::from_cols_array_2d(&[ + [1f32, 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., -1., 0.], + [0., 0., 1., 1.], + ]); + + return z_reversal * Mat4::from_cols_array(&cols); + } else { + // normal projection + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -(far_z + offset_z) / (far_z - near_z); + cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + } + + Mat4::from_cols_array(&cols) + } + + fn update(&mut self, _width: f32, _height: f32) {} + + fn far(&self) -> f32 { + self.far + } + + fn get_frustum_corners(&self, z_near: f32, z_far: f32) -> [Vec3A; 8] { + let tan_angle_left = self.fov.angle_left.tan(); + let tan_angle_right = self.fov.angle_right.tan(); + + let tan_angle_bottom = self.fov.angle_down.tan(); + let tan_angle_top = self.fov.angle_up.tan(); + + // NOTE: These vertices are in the specific order required by [`calculate_cascade`]. + [ + Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_near, // bottom right + Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_near, // top right + Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_near, // top left + Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_near, // bottom left + Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_far, // bottom right + Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_far, // top right + Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_far, // top left + Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_far, // bottom left + ] + } +} + +pub fn xr_camera_head_sync( + views: ResMut, + mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, +) { + //TODO calculate HMD position + for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { + let view_idx = match camera_type { + XrCameraType::Xr(eye) => *eye as usize, + // I don't belive we need a flatscrenn cam, that's just a cam without this component + XrCameraType::Flatscreen => continue, + }; + let view = match views.get(view_idx) { + Some(views) => views, + None => continue, + }; + xr_projection.fov = view.fov; + transform.rotation = view.pose.orientation.to_quat(); + transform.translation = view.pose.position.to_vec3(); + } +} diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs new file mode 100644 index 0000000..d7db1c9 --- /dev/null +++ b/src/xr_init/mod.rs @@ -0,0 +1,197 @@ +pub mod schedules; +pub use schedules::*; + +use bevy::{ + prelude::*, + render::{ + camera::{ManualTextureView, ManualTextureViews}, + extract_resource::{ExtractResource, ExtractResourcePlugin}, + renderer::{RenderAdapter, RenderDevice, RenderInstance}, + }, + window::{PrimaryWindow, RawHandleWrapper}, +}; + +use crate::{ + graphics, + resources::{ + OXrSessionSetupInfo, XrFormat, XrInstance, XrResolution, XrSession, XrSessionRunning, + XrSwapchain, + }, + LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE, +}; + +#[derive(Resource, Event, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource)] +pub enum XrStatus { + NoInstance, + Enabled, + Enabling, + Disabled, + Disabling, +} + +#[derive( + Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut, +)] +pub struct XrShouldRender(bool); + +pub struct XrEarlyInitPlugin; + +pub struct XrInitPlugin; + +pub fn xr_only() -> impl FnMut(Option>) -> bool { + resource_exists_and_equals(XrStatus::Enabled) +} +pub fn xr_render_only() -> impl FnMut(Option>) -> bool { + resource_exists_and_equals(XrShouldRender(true)) +} + +impl Plugin for XrEarlyInitPlugin { + fn build(&self, app: &mut App) { + app.add_event::() + .add_event::() + .add_event::() + .add_event::(); + } +} + +impl Plugin for XrInitPlugin { + fn build(&self, app: &mut App) { + add_schedules(app); + app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); + app.init_resource::(); + app.add_systems(PreUpdate, setup_xr.run_if(on_event::())) + .add_systems(PreUpdate, cleanup_xr.run_if(on_event::())); + app.add_systems( + PostUpdate, + start_xr_session.run_if(on_event::()), + ); + app.add_systems( + PostUpdate, + stop_xr_session.run_if(on_event::()), + ); + app.add_systems(XrSetup, setup_manual_texture_views); + } +} + +fn setup_manual_texture_views( + mut manual_texture_views: ResMut, + swapchain: Res, + xr_resolution: Res, + xr_format: Res, +) { + info!("Creating Texture views"); + let (left, right) = swapchain.get_render_views(); + let left = ManualTextureView { + texture_view: left.into(), + size: **xr_resolution, + format: **xr_format, + }; + let right = ManualTextureView { + texture_view: right.into(), + size: **xr_resolution, + format: **xr_format, + }; + manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); + manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); +} + +pub fn setup_xr(world: &mut World) { + world.run_schedule(XrPreSetup); + world.run_schedule(XrSetup); + world.run_schedule(XrPrePostSetup); + world.run_schedule(XrPostSetup); + *world.resource_mut::() = XrStatus::Enabled; +} +fn cleanup_xr(world: &mut World) { + world.run_schedule(XrPreCleanup); + world.run_schedule(XrCleanup); + world.run_schedule(XrPostCleanup); + *world.resource_mut::() = XrStatus::Disabled; +} + +#[derive(Event, Clone, Copy, Default)] +pub struct StartXrSession; + +#[derive(Event, Clone, Copy, Default)] +pub struct EndXrSession; + +#[derive(Event, Clone, Copy, Default)] +struct SetupXrData; +#[derive(Event, Clone, Copy, Default)] +pub(crate) struct CleanupXrData; + +#[allow(clippy::too_many_arguments)] +fn start_xr_session( + mut commands: Commands, + mut setup_xr: EventWriter, + mut status: ResMut, + instance: Res, + primary_window: Query<&RawHandleWrapper, With>, + setup_info: NonSend, + render_device: Res, + render_adapter: Res, + render_instance: Res, +) { + info!("start Session"); + match *status { + XrStatus::Disabled => {} + XrStatus::NoInstance => { + warn!("Trying to start OpenXR Session without instance, ignoring"); + return; + } + XrStatus::Enabled | XrStatus::Enabling => { + warn!("Trying to start OpenXR Session while one already exists, ignoring"); + return; + } + XrStatus::Disabling => { + warn!("Trying to start OpenXR Session while one is stopping, ignoring"); + return; + } + } + let ( + xr_session, + xr_resolution, + xr_format, + xr_session_running, + xr_frame_waiter, + xr_swapchain, + xr_input, + xr_views, + xr_frame_state, + ) = match graphics::start_xr_session( + primary_window.get_single().cloned().ok(), + &setup_info, + &instance, + &render_device, + &render_adapter, + &render_instance, + ) { + Ok(data) => data, + Err(err) => { + error!("Unable to start OpenXR Session: {}", err); + return; + } + }; + commands.insert_resource(xr_session); + commands.insert_resource(xr_resolution); + commands.insert_resource(xr_format); + commands.insert_resource(xr_session_running); + commands.insert_resource(xr_frame_waiter); + commands.insert_resource(xr_swapchain); + commands.insert_resource(xr_input); + commands.insert_resource(xr_views); + commands.insert_resource(xr_frame_state); + *status = XrStatus::Enabling; + setup_xr.send_default(); +} + +fn stop_xr_session(session: ResMut, mut status: ResMut) { + match session.request_exit() { + Ok(_) => {} + Err(err) => { + error!("Error while trying to request session exit: {}", err) + } + } + *status = XrStatus::Enabling; +} diff --git a/src/xr_init/schedules.rs b/src/xr_init/schedules.rs new file mode 100644 index 0000000..f08fde6 --- /dev/null +++ b/src/xr_init/schedules.rs @@ -0,0 +1,51 @@ +use bevy::{ecs::schedule::{ScheduleLabel, Schedule, ExecutorKind}, app::App}; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPreSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPrePostSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPostSetup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPreCleanup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrCleanup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPostCleanup; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPreRenderUpdate; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrRenderUpdate; + +#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)] +pub struct XrPostRenderUpdate; + +pub(super) fn add_schedules(app: &mut App) { + let schedules = [ + Schedule::new(XrPreSetup), + Schedule::new(XrSetup), + Schedule::new(XrPrePostSetup), + Schedule::new(XrPostSetup), + Schedule::new(XrPreRenderUpdate), + Schedule::new(XrRenderUpdate), + Schedule::new(XrPostRenderUpdate), + Schedule::new(XrPreCleanup), + Schedule::new(XrCleanup), + Schedule::new(XrPostCleanup), + ]; + for mut schedule in schedules { + schedule.set_executor_kind(ExecutorKind::SingleThreaded); + schedule.set_apply_final_deferred(true); + app.add_schedule(schedule); + } +} From db5539cfd9e5538176028dda35b23a76b54152ec Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 23 Jan 2024 06:10:01 +0100 Subject: [PATCH 07/25] removed weird file --- + | 294 -------------------------------------------------------------- 1 file changed, 294 deletions(-) delete mode 100644 + diff --git a/+ b/+ deleted file mode 100644 index 8e33f19..0000000 --- a/+ +++ /dev/null @@ -1,294 +0,0 @@ -use crate::xr_input::{QuatConv, Vec3Conv}; -use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; -use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; -use bevy::ecs::system::lifetimeless::Read; -use bevy::math::Vec3A; -use bevy::prelude::*; -use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; -use bevy::render::extract_component::ExtractComponent; -use bevy::render::primitives::Frustum; -use bevy::render::view::{ColorGrading, VisibleEntities}; -use openxr::Fovf; - -#[derive(Bundle)] -pub struct XrCamerasBundle { - pub left: XrCameraBundle, - pub right: XrCameraBundle, -} -impl XrCamerasBundle { - pub fn new() -> Self { - Self::default() - } -} -impl Default for XrCamerasBundle { - fn default() -> Self { - Self { - left: XrCameraBundle::new(Eye::Left), - right: XrCameraBundle::new(Eye::Right), - } - } -} - -#[derive(Bundle)] -pub struct XrCameraBundle { - pub camera: Camera, - pub camera_render_graph: CameraRenderGraph, - pub xr_projection: XRProjection, - pub visible_entities: VisibleEntities, - pub frustum: Frustum, - pub transform: Transform, - pub global_transform: GlobalTransform, - pub camera_3d: Camera3d, - pub tonemapping: Tonemapping, - pub dither: DebandDither, - pub color_grading: ColorGrading, - pub xr_camera_type: XrCameraType, -} -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] -pub enum XrCameraType { - Xr(Eye), - Flatscreen, -} - - -#[derive(Component)] -pub(super) struct TransformExtract; - - -impl ExtractComponent for TransformExtract { - type Query = Read; - - type Filter = (); - - type Out = Transform; - - fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { - info!("extracting Transform"); - Some(*item) - } -} - -impl ExtractComponent for XrCameraType { - type Query = Read; - - type Filter = (); - - type Out = Self; - - fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { - info!("extracting Cam Type"); - Some(*item) - } -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub enum Eye { - Left = 0, - Right = 1, -} - -impl XrCameraBundle { - pub fn new(eye: Eye) -> Self { - Self { - camera: Camera { - order: -1, - target: RenderTarget::TextureView(match eye { - Eye::Left => LEFT_XR_TEXTURE_HANDLE, - Eye::Right => RIGHT_XR_TEXTURE_HANDLE, - }), - viewport: None, - ..default() - }, - camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME), - xr_projection: Default::default(), - visible_entities: Default::default(), - frustum: Default::default(), - transform: Default::default(), - global_transform: Default::default(), - camera_3d: Default::default(), - tonemapping: Default::default(), - dither: DebandDither::Enabled, - color_grading: Default::default(), - xr_camera_type: XrCameraType::Xr(eye), - } - } -} - -#[derive(Debug, Clone, Component, Reflect, ExtractComponent)] -#[reflect(Component, Default)] -pub struct XRProjection { - pub near: f32, - pub far: f32, - #[reflect(ignore)] - pub fov: Fovf, -} - -impl Default for XRProjection { - fn default() -> Self { - Self { - near: 0.1, - far: 1000., - fov: Default::default(), - } - } -} - -impl XRProjection { - pub fn new(near: f32, far: f32, fov: Fovf) -> Self { - XRProjection { near, far, fov } - } -} - -impl CameraProjection for XRProjection { - // ============================================================================= - // math code adapted from - // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h - // Copyright (c) 2017 The Khronos Group Inc. - // Copyright (c) 2016 Oculus VR, LLC. - // SPDX-License-Identifier: Apache-2.0 - // ============================================================================= - fn get_projection_matrix(&self) -> Mat4 { - // symmetric perspective for debugging - // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); - // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); - // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); - - let fov = self.fov; - let is_vulkan_api = false; // FIXME wgpu probably abstracts this - let near_z = self.near; - let far_z = -1.; // use infinite proj - // let far_z = self.far; - - let tan_angle_left = fov.angle_left.tan(); - let tan_angle_right = fov.angle_right.tan(); - - let tan_angle_down = fov.angle_down.tan(); - let tan_angle_up = fov.angle_up.tan(); - - let tan_angle_width = tan_angle_right - tan_angle_left; - - // Set to tanAngleDown - tanAngleUp for a clip space with positive Y - // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with - // positive Y up (OpenGL / D3D / Metal). - // const float tanAngleHeight = - // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); - let tan_angle_height = if is_vulkan_api { - tan_angle_down - tan_angle_up - } else { - tan_angle_up - tan_angle_down - }; - - // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). - // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). - // const float offsetZ = - // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; - // FIXME handle enum of graphics apis - let offset_z = 0.; - - let mut cols: [f32; 16] = [0.0; 16]; - - if far_z <= near_z { - // place the far plane at infinity - cols[0] = 2. / tan_angle_width; - cols[4] = 0.; - cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; - cols[12] = 0.; - - cols[1] = 0.; - cols[5] = 2. / tan_angle_height; - cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; - cols[13] = 0.; - - cols[2] = 0.; - cols[6] = 0.; - cols[10] = -1.; - cols[14] = -(near_z + offset_z); - - cols[3] = 0.; - cols[7] = 0.; - cols[11] = -1.; - cols[15] = 0.; - - // bevy uses the _reverse_ infinite projection - // https://dev.theomader.com/depth-precision/ - let z_reversal = Mat4::from_cols_array_2d(&[ - [1f32, 0., 0., 0.], - [0., 1., 0., 0.], - [0., 0., -1., 0.], - [0., 0., 1., 1.], - ]); - - return z_reversal * Mat4::from_cols_array(&cols); - } else { - // normal projection - cols[0] = 2. / tan_angle_width; - cols[4] = 0.; - cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; - cols[12] = 0.; - - cols[1] = 0.; - cols[5] = 2. / tan_angle_height; - cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; - cols[13] = 0.; - - cols[2] = 0.; - cols[6] = 0.; - cols[10] = -(far_z + offset_z) / (far_z - near_z); - cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); - - cols[3] = 0.; - cols[7] = 0.; - cols[11] = -1.; - cols[15] = 0.; - } - - Mat4::from_cols_array(&cols) - } - - fn update(&mut self, _width: f32, _height: f32) {} - - fn far(&self) -> f32 { - self.far - } - - fn get_frustum_corners(&self, z_near: f32, z_far: f32) -> [Vec3A; 8] { - let tan_angle_left = self.fov.angle_left.tan(); - let tan_angle_right = self.fov.angle_right.tan(); - - let tan_angle_bottom = self.fov.angle_down.tan(); - let tan_angle_top = self.fov.angle_up.tan(); - - // NOTE: These vertices are in the specific order required by [`calculate_cascade`]. - [ - Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_near, // bottom right - Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_near, // top right - Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_near, // top left - Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_near, // bottom left - Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_far, // bottom right - Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_far, // top right - Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_far, // top left - Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_far, // bottom left - ] - } -} - -pub fn xr_camera_head_sync( - views: ResMut, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, -) { - //TODO calculate HMD position - for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - XrCameraType::Xr(eye) => *eye as usize, - // I don't belive we need a flatscrenn cam, that's just a cam without this component - XrCameraType::Flatscreen => continue, - }; - let view = match views.get(view_idx) { - Some(views) => views, - None => continue, - }; - xr_projection.fov = view.fov; - transform.rotation = view.pose.orientation.to_quat(); - transform.translation = view.pose.position.to_vec3(); - } -} From bc1985eeddcb33cbbb9e1174103533b0cb8178a1 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Wed, 24 Jan 2024 00:28:53 +0100 Subject: [PATCH 08/25] works on quest broken on wivrn --- src/lib.rs | 70 ++++++++++++++++++------------------------------------ 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3b52be6..3b6b4fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,6 @@ pub mod xr_input; use std::fs::File; use std::io::{BufWriter, Write}; -use std::net::TcpStream; use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; @@ -21,9 +20,10 @@ use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; -use bevy::render::renderer::RenderInstance; +use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; +use bevy::transform::systems::{propagate_transforms, sync_simple_transforms}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; @@ -142,16 +142,25 @@ impl Plugin for OpenXrPlugin { xr_pre_frame .run_if(xr_only()) .run_if(xr_render_only()) - .in_set(RenderSet::Prepare), - ); - render_app.add_systems( - Render, - (locate_views, xr_input::xr_camera::xr_camera_head_sync) - .chain() - .run_if(xr_only()) - .run_if(xr_render_only()) - .in_set(RenderSet::Prepare), + // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet + // right before rendering + .before(render_system) + .after(RenderSet::ExtractCommands), + // .in_set(RenderSet::Prepare), ); + // render_app.add_systems( + // Render, + // ( + // locate_views, + // xr_input::xr_camera::xr_camera_head_sync, + // sync_simple_transforms, + // propagate_transforms, + // ) + // .chain() + // .run_if(xr_only()) + // .run_if(xr_render_only()) + // .in_set(RenderSet::Prepare), + // ); render_app.add_systems( Render, xr_end_frame @@ -159,15 +168,9 @@ impl Plugin for OpenXrPlugin { .run_if(xr_render_only()) .after(RenderSet::Render), ); - render_app.insert_resource(TcpConnection( - TcpStream::connect("192.168.2.100:6969").unwrap(), - )); } } -#[derive(Resource)] -struct TcpConnection(TcpStream); - #[derive(Default)] pub struct DefaultXrPlugins { pub reqeusted_extensions: XrExtensions, @@ -180,7 +183,7 @@ impl PluginGroup for DefaultXrPlugins { DefaultPlugins .build() .disable::() - .disable::() + // .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, reqeusted_extensions: self.reqeusted_extensions, @@ -273,6 +276,7 @@ pub fn xr_wait_frame( ) { { let _span = info_span!("xr_wait_frame").entered(); + info!("Pre Frame Wait"); *frame_state = match frame_waiter.wait() { Ok(a) => a.into(), Err(e) => { @@ -280,6 +284,7 @@ pub fn xr_wait_frame( return; } }; + info!("Post Frame Wait"); **should_render = frame_state.should_render; } } @@ -296,9 +301,7 @@ pub fn xr_pre_frame( } { let _span = info_span!("xr_wait_image").entered(); - info!("wait image"); swapchain.wait_image().unwrap(); - info!("waited image"); } { let _span = info_span!("xr_update_manual_texture_views").entered(); @@ -325,8 +328,6 @@ pub fn xr_end_frame( swapchain: Res, resolution: Res, environment_blend_mode: Res, - mut connection: ResMut, - cams: Query<(&Transform, &XrCameraType)>, ) { #[cfg(target_os = "android")] { @@ -338,31 +339,6 @@ pub fn xr_end_frame( let _span = info_span!("xr_release_image").entered(); swapchain.release_image().unwrap(); } - let mut cam = None; - for (t, c) in &cams { - if *c == XrCameraType::Xr(xr_input::xr_camera::Eye::Left) { - cam = Some(*t); - break; - } - } - let _ = std::writeln!( - &mut connection.0, - "{},{},{},{},{},{},{},{},{},{},{},{},{},{}", - views[0].pose.position.x, - views[0].pose.position.y, - views[0].pose.position.z, - views[0].pose.orientation.x, - views[0].pose.orientation.y, - views[0].pose.orientation.z, - views[0].pose.orientation.w, - cam.unwrap().translation.x, - cam.unwrap().translation.y, - cam.unwrap().translation.z, - cam.unwrap().rotation.x, - cam.unwrap().rotation.y, - cam.unwrap().rotation.z, - cam.unwrap().rotation.w, - ); { let _span = info_span!("xr_end_frame").entered(); let result = swapchain.end( From a3c33cb8b6928ebc9990c1b711cf3086e2264b39 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Wed, 24 Jan 2024 03:46:20 +0100 Subject: [PATCH 09/25] win + steamvr and quest working, wivrn still broken --- src/lib.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3b6b4fe..f5675b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,7 @@ use input::XrInput; use openxr as xr; // use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; use resources::*; -use xr::FormFactor; +use xr::{FormFactor, FrameState}; use xr_init::{ xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, }; @@ -119,9 +119,7 @@ impl Plugin for OpenXrPlugin { PreUpdate, ( xr_reset_should_render, - apply_deferred, xr_wait_frame.run_if(xr_only()), - apply_deferred, locate_views.run_if(xr_only()), apply_deferred, ) @@ -146,7 +144,7 @@ impl Plugin for OpenXrPlugin { // right before rendering .before(render_system) .after(RenderSet::ExtractCommands), - // .in_set(RenderSet::Prepare), + // .in_set(RenderSet::Prepare), ); // render_app.add_systems( // Render, @@ -284,6 +282,13 @@ pub fn xr_wait_frame( return; } }; + #[allow(clippy::erasing_op)] + { + frame_state.predicted_display_time = xr::Time::from_nanos( + frame_state.predicted_display_time.as_nanos() + + (frame_state.predicted_display_period.as_nanos() * 1), + ); + }; info!("Post Frame Wait"); **should_render = frame_state.should_render; } From 9d6d60cbe73976e7ab08ea344167e7205e1711fc Mon Sep 17 00:00:00 2001 From: Schmarni Date: Thu, 25 Jan 2024 06:39:44 +0100 Subject: [PATCH 10/25] implement simple pipelined rendering --- src/lib.rs | 45 ++++++++++++++++++++++++++++++++++++--------- src/xr_init/mod.rs | 17 +++++++++++++---- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f5675b7..2659469 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ use openxr as xr; use resources::*; use xr::{FormFactor, FrameState}; use xr_init::{ - xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, + xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, XrHasWaited, xr_after_wait_only, }; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; @@ -118,7 +118,7 @@ impl Plugin for OpenXrPlugin { app.add_systems( PreUpdate, ( - xr_reset_should_render, + xr_reset_per_frame_resources, xr_wait_frame.run_if(xr_only()), locate_views.run_if(xr_only()), apply_deferred, @@ -130,15 +130,15 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_begin_frame - .run_if(xr_only()) - .run_if(xr_render_only()) + .run_if(xr_only()).run_if(xr_after_wait_only()) + // .run_if(xr_render_only()) .after(RenderSet::ExtractCommands) .before(xr_pre_frame), ); render_app.add_systems( Render, xr_pre_frame - .run_if(xr_only()) + .run_if(xr_only()).run_if(xr_after_wait_only()) .run_if(xr_render_only()) // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet // right before rendering @@ -162,10 +162,34 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_end_frame - .run_if(xr_only()) + .run_if(xr_only()).run_if(xr_after_wait_only()) .run_if(xr_render_only()) .after(RenderSet::Render), ); + render_app.add_systems( + Render, + xr_skip_frame + .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(not(xr_render_only())) + .after(RenderSet::Render), + ); + } +} + +fn xr_skip_frame( + xr_swapchain: Res, + xr_frame_state: Res, + environment_blend_mode: Res, +) { + let swapchain: &Swapchain = &xr_swapchain; + // swapchain.begin().unwrap(); + match swapchain { + Swapchain::Vulkan(swap) => { + swap.stream + .lock() + .unwrap() + .end(xr_frame_state.predicted_display_time, **environment_blend_mode, &[]).unwrap(); + } } } @@ -180,8 +204,8 @@ impl PluginGroup for DefaultXrPlugins { fn build(self) -> PluginGroupBuilder { DefaultPlugins .build() + .disable::() .disable::() - // .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, reqeusted_extensions: self.reqeusted_extensions, @@ -211,8 +235,9 @@ impl PluginGroup for DefaultXrPlugins { } } -fn xr_reset_should_render(mut should: ResMut) { +fn xr_reset_per_frame_resources(mut should: ResMut,mut waited: ResMut) { **should = false; + **waited = false; } fn xr_poll_events( @@ -271,6 +296,7 @@ pub fn xr_wait_frame( mut frame_state: ResMut, mut frame_waiter: ResMut, mut should_render: ResMut, + mut waited: ResMut, ) { { let _span = info_span!("xr_wait_frame").entered(); @@ -286,11 +312,12 @@ pub fn xr_wait_frame( { frame_state.predicted_display_time = xr::Time::from_nanos( frame_state.predicted_display_time.as_nanos() - + (frame_state.predicted_display_period.as_nanos() * 1), + + (frame_state.predicted_display_period.as_nanos() * 0), ); }; info!("Post Frame Wait"); **should_render = frame_state.should_render; + **waited = true; } } diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs index d7db1c9..06d9578 100644 --- a/src/xr_init/mod.rs +++ b/src/xr_init/mod.rs @@ -33,16 +33,23 @@ pub enum XrStatus { Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut, )] pub struct XrShouldRender(bool); +#[derive( + Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut, +)] +pub struct XrHasWaited(bool); pub struct XrEarlyInitPlugin; pub struct XrInitPlugin; -pub fn xr_only() -> impl FnMut(Option>) -> bool { - resource_exists_and_equals(XrStatus::Enabled) +pub fn xr_only() -> impl FnMut(Res) -> bool { + resource_equals(XrStatus::Enabled) } -pub fn xr_render_only() -> impl FnMut(Option>) -> bool { - resource_exists_and_equals(XrShouldRender(true)) +pub fn xr_render_only() -> impl FnMut(Res) -> bool { + resource_equals(XrShouldRender(true)) +} +pub fn xr_after_wait_only() -> impl FnMut(Res) -> bool { + resource_equals(XrHasWaited(true)) } impl Plugin for XrEarlyInitPlugin { @@ -59,7 +66,9 @@ impl Plugin for XrInitPlugin { add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); + app.add_plugins(ExtractResourcePlugin::::default()); app.init_resource::(); + app.init_resource::(); app.add_systems(PreUpdate, setup_xr.run_if(on_event::())) .add_systems(PreUpdate, cleanup_xr.run_if(on_event::())); app.add_systems( From ac7c1089d954cc60c48652aae25dfacaeaf1e029 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Fri, 26 Jan 2024 04:35:51 +0100 Subject: [PATCH 11/25] fixed pipelined rendering on quest 2 --- src/lib.rs | 81 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2659469..8d77c30 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ use crate::xr_init::{StartXrSession, XrInitPlugin}; use crate::xr_input::hands::hand_tracking::DisableHandTracking; use crate::xr_input::oculus_touch::ActionSets; use bevy::app::{AppExit, PluginGroupBuilder}; +use bevy::core::TaskPoolThreadAssignmentPolicy; use bevy::ecs::system::SystemState; use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; @@ -23,6 +24,7 @@ use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; +use bevy::tasks::available_parallelism; use bevy::transform::systems::{propagate_transforms, sync_simple_transforms}; use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper}; use graphics::extensions::XrExtensions; @@ -33,7 +35,8 @@ use openxr as xr; use resources::*; use xr::{FormFactor, FrameState}; use xr_init::{ - xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrShouldRender, XrStatus, XrHasWaited, xr_after_wait_only, + xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrHasWaited, + XrShouldRender, XrStatus, }; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; @@ -130,7 +133,8 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_begin_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) // .run_if(xr_render_only()) .after(RenderSet::ExtractCommands) .before(xr_pre_frame), @@ -138,7 +142,8 @@ impl Plugin for OpenXrPlugin { render_app.add_systems( Render, xr_pre_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) .run_if(xr_render_only()) // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet // right before rendering @@ -146,30 +151,32 @@ impl Plugin for OpenXrPlugin { .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), ); - // render_app.add_systems( - // Render, - // ( - // locate_views, - // xr_input::xr_camera::xr_camera_head_sync, - // sync_simple_transforms, - // propagate_transforms, - // ) - // .chain() - // .run_if(xr_only()) - // .run_if(xr_render_only()) - // .in_set(RenderSet::Prepare), - // ); + render_app.add_systems( + Render, + ( + locate_views, + xr_input::xr_camera::xr_camera_head_sync, + sync_simple_transforms, + propagate_transforms, + ) + .chain() + .run_if(xr_only()) + // .run_if(xr_render_only()) + .in_set(RenderSet::Prepare), + ); render_app.add_systems( Render, xr_end_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) .run_if(xr_render_only()) .after(RenderSet::Render), ); render_app.add_systems( Render, xr_skip_frame - .run_if(xr_only()).run_if(xr_after_wait_only()) + .run_if(xr_only()) + .run_if(xr_after_wait_only()) .run_if(not(xr_render_only())) .after(RenderSet::Render), ); @@ -188,7 +195,12 @@ fn xr_skip_frame( swap.stream .lock() .unwrap() - .end(xr_frame_state.predicted_display_time, **environment_blend_mode, &[]).unwrap(); + .end( + xr_frame_state.predicted_display_time, + **environment_blend_mode, + &[], + ) + .unwrap(); } } } @@ -204,7 +216,20 @@ impl PluginGroup for DefaultXrPlugins { fn build(self) -> PluginGroupBuilder { DefaultPlugins .build() - .disable::() + .set(TaskPoolPlugin { + task_pool_options: TaskPoolOptions { + compute: TaskPoolThreadAssignmentPolicy { + // set the minimum # of compute threads + // to the total number of available threads + min_threads: 2, + max_threads: std::usize::MAX, // unlimited max threads + percent: 1.0, // this value is irrelevant in this case + }, + // keep the defaults for everything else + ..default() + }, + }) + // .disable::() .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, @@ -235,7 +260,10 @@ impl PluginGroup for DefaultXrPlugins { } } -fn xr_reset_per_frame_resources(mut should: ResMut,mut waited: ResMut) { +fn xr_reset_per_frame_resources( + mut should: ResMut, + mut waited: ResMut, +) { **should = false; **waited = false; } @@ -308,13 +336,10 @@ pub fn xr_wait_frame( return; } }; - #[allow(clippy::erasing_op)] - { - frame_state.predicted_display_time = xr::Time::from_nanos( - frame_state.predicted_display_time.as_nanos() - + (frame_state.predicted_display_period.as_nanos() * 0), - ); - }; + frame_state.predicted_display_time = xr::Time::from_nanos( + frame_state.predicted_display_time.as_nanos() + + frame_state.predicted_display_period.as_nanos(), + ); info!("Post Frame Wait"); **should_render = frame_state.should_render; **waited = true; From 1e4db18f84f35bc2c22ef2d7337dc1cef6abb46c Mon Sep 17 00:00:00 2001 From: Schmarni Date: Sat, 27 Jan 2024 04:06:27 +0100 Subject: [PATCH 12/25] commit --- src/lib.rs | 12 ++++++------ src/xr_input/xr_camera.rs | 16 +--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8d77c30..44fc206 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,9 @@ use bevy::app::{AppExit, PluginGroupBuilder}; use bevy::core::TaskPoolThreadAssignmentPolicy; use bevy::ecs::system::SystemState; use bevy::prelude::*; -use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; +use bevy::render::camera::{ + camera_system, ManualTextureView, ManualTextureViewHandle, ManualTextureViews, +}; use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; @@ -42,7 +44,7 @@ use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin}; use xr_input::hands::XrHandPlugins; -use xr_input::xr_camera::XrCameraType; +use xr_input::xr_camera::{XRProjection, XrCameraType}; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -170,7 +172,7 @@ impl Plugin for OpenXrPlugin { .run_if(xr_only()) .run_if(xr_after_wait_only()) .run_if(xr_render_only()) - .after(RenderSet::Render), + .in_set(RenderSet::Cleanup), ); render_app.add_systems( Render, @@ -178,7 +180,7 @@ impl Plugin for OpenXrPlugin { .run_if(xr_only()) .run_if(xr_after_wait_only()) .run_if(not(xr_render_only())) - .after(RenderSet::Render), + .in_set(RenderSet::Cleanup), ); } } @@ -219,8 +221,6 @@ impl PluginGroup for DefaultXrPlugins { .set(TaskPoolPlugin { task_pool_options: TaskPoolOptions { compute: TaskPoolThreadAssignmentPolicy { - // set the minimum # of compute threads - // to the total number of available threads min_threads: 2, max_threads: std::usize::MAX, // unlimited max threads percent: 1.0, // this value is irrelevant in this case diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 1d143c3..95b70b1 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -44,17 +44,15 @@ pub struct XrCameraBundle { pub color_grading: ColorGrading, pub xr_camera_type: XrCameraType, } -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)] pub enum XrCameraType { Xr(Eye), Flatscreen, } - #[derive(Component)] pub(super) struct TransformExtract; - impl ExtractComponent for TransformExtract { type Query = Read; @@ -67,18 +65,6 @@ impl ExtractComponent for TransformExtract { } } -impl ExtractComponent for XrCameraType { - type Query = Read; - - type Filter = (); - - type Out = Self; - - fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { - Some(*item) - } -} - #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, From 5c81f135e792a79493c0cc279bfda50331fd48fc Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 29 Jan 2024 04:17:20 +0100 Subject: [PATCH 13/25] fix late latching? --- src/lib.rs | 8 ++++++++ src/xr_input/xr_camera.rs | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 44fc206..80382aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ use bevy::render::extract_resource::ExtractResourcePlugin; use bevy::render::pipelined_rendering::PipelinedRenderingPlugin; use bevy::render::renderer::{render_system, RenderInstance}; use bevy::render::settings::RenderCreation; +use bevy::render::view::ExtractedView; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::tasks::available_parallelism; use bevy::transform::systems::{propagate_transforms, sync_simple_transforms}; @@ -160,6 +161,7 @@ impl Plugin for OpenXrPlugin { xr_input::xr_camera::xr_camera_head_sync, sync_simple_transforms, propagate_transforms, + update_cam_views, ) .chain() .run_if(xr_only()) @@ -185,6 +187,12 @@ impl Plugin for OpenXrPlugin { } } +fn update_cam_views(mut query: Query<(&mut ExtractedView, &GlobalTransform)>) { + for (mut view, transform) in &mut query { + view.transform = *transform; + } +} + fn xr_skip_frame( xr_swapchain: Res, xr_frame_state: Res, diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 95b70b1..912b3ec 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -65,6 +65,20 @@ impl ExtractComponent for TransformExtract { } } +#[derive(Component)] +pub(super) struct GlobalTransformExtract; + +impl ExtractComponent for GlobalTransformExtract { + type Query = Read; + + type Filter = (); + + type Out = GlobalTransform; + + fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option { + Some(*item) + } +} #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, From 9704607c8c9ed19ac8d6c9f9c5372fbbe68c8100 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 5 Feb 2024 02:27:27 +0100 Subject: [PATCH 14/25] stuff --- src/graphics/vulkan.rs | 4 +++- src/lib.rs | 43 +++++++++++++++++++++++---------------- src/resource_macros.rs | 29 ++++++++++++++++++++++++++ src/resources.rs | 18 +++++++++++++--- src/xr_input/mod.rs | 3 ++- src/xr_input/xr_camera.rs | 27 ++++++++++++++++++++++-- 6 files changed, 100 insertions(+), 24 deletions(-) diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index ac0948d..f4ca0f0 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -356,7 +356,7 @@ pub fn start_xr_session( }); let swapchain_format = surface .as_ref() - .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0]) + .map(|surface| surface.get_capabilities(wgpu_adapter).formats[0]) .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb); let resolution = uvec2( @@ -387,6 +387,7 @@ pub fn start_xr_session( let buffers = images .into_iter() .map(|color_image| { + info!("image map swapchain"); let color_image = vk::Image::from_raw(color_image); let wgpu_hal_texture = unsafe { ::Device::texture_from_raw( @@ -449,6 +450,7 @@ pub fn start_xr_session( .into(), XrInput::new(xr_instance, &session.into_any_graphics())?, Vec::default().into(), + // Feels wrong to return a FrameState here, we probably should just wait for the next frame xr::FrameState { predicted_display_time: xr::Time::from_nanos(1), predicted_display_period: xr::Duration::from_nanos(1), diff --git a/src/lib.rs b/src/lib.rs index 80382aa..83b09f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -154,20 +154,20 @@ impl Plugin for OpenXrPlugin { .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), ); - render_app.add_systems( - Render, - ( - locate_views, - xr_input::xr_camera::xr_camera_head_sync, - sync_simple_transforms, - propagate_transforms, - update_cam_views, - ) - .chain() - .run_if(xr_only()) - // .run_if(xr_render_only()) - .in_set(RenderSet::Prepare), - ); + // render_app.add_systems( + // Render, + // ( + // locate_views, + // xr_input::xr_camera::xr_camera_head_sync_render, + // // sync_simple_transforms, + // // propagate_transforms, + // // update_cam_views, + // ) + // .chain() + // .run_if(xr_only()) + // // .run_if(xr_render_only()) + // .in_set(RenderSet::Prepare), + // ); render_app.add_systems( Render, xr_end_frame @@ -187,6 +187,8 @@ impl Plugin for OpenXrPlugin { } } +// Confirmed Working +// Not Working Actually, the cam doesn't render with the new pose for some reason fn update_cam_views(mut query: Query<(&mut ExtractedView, &GlobalTransform)>) { for (mut view, transform) in &mut query { view.transform = *transform; @@ -199,7 +201,6 @@ fn xr_skip_frame( environment_blend_mode: Res, ) { let swapchain: &Swapchain = &xr_swapchain; - // swapchain.begin().unwrap(); match swapchain { Swapchain::Vulkan(swap) => { swap.stream @@ -344,10 +345,14 @@ pub fn xr_wait_frame( return; } }; - frame_state.predicted_display_time = xr::Time::from_nanos( + info!( + "Post Wait Time: {}", frame_state.predicted_display_time.as_nanos() - + frame_state.predicted_display_period.as_nanos(), ); + // frame_state.predicted_display_time = xr::Time::from_nanos( + // frame_state.predicted_display_time.as_nanos() + // + frame_state.predicted_display_period.as_nanos(), + // ); info!("Post Frame Wait"); **should_render = frame_state.should_render; **waited = true; @@ -406,6 +411,10 @@ pub fn xr_end_frame( } { let _span = info_span!("xr_end_frame").entered(); + info!( + "End Frame Time: {}", + xr_frame_state.predicted_display_time.as_nanos() + ); let result = swapchain.end( xr_frame_state.predicted_display_time, &views, diff --git a/src/resource_macros.rs b/src/resource_macros.rs index 735ec50..bb83a63 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -32,6 +32,35 @@ macro_rules! xr_resource_wrapper { }; } +#[macro_export] +macro_rules! xr_resource_wrapper_no_extract { + ($wrapper_type:ident, $xr_type:ty) => { + #[derive( + Clone, Copy, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut, + )] + pub struct $wrapper_type($xr_type); + + impl $wrapper_type { + pub fn new(value: $xr_type) -> Self { + Self(value) + } + } + + // impl std::ops::Deref for $wrapper_type { + // type Target = $xr_type; + // + // fn deref(&self) -> &Self::Target { + // &self.0 + // } + // } + + impl From<$xr_type> for $wrapper_type { + fn from(value: $xr_type) -> Self { + Self::new(value) + } + } + }; +} #[macro_export] macro_rules! xr_arc_resource_wrapper { ($wrapper_type:ident, $xr_type:ty) => { diff --git a/src/resources.rs b/src/resources.rs index da72cf6..2854abc 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -4,10 +4,10 @@ use std::sync::Mutex; use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; -use crate::resource_macros::*; use crate::xr_init::XrStatus; +use crate::{resource_macros::*, xr_resource_wrapper_no_extract}; use bevy::prelude::*; -use bevy::render::extract_resource::ExtractResourcePlugin; +use bevy::render::extract_resource::{ExtractResource, ExtractResourcePlugin}; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); @@ -15,12 +15,24 @@ xr_resource_wrapper!(XrSession, xr::Session); xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); xr_resource_wrapper!(XrResolution, UVec2); xr_resource_wrapper!(XrFormat, wgpu::TextureFormat); -xr_resource_wrapper!(XrFrameState, xr::FrameState); +xr_resource_wrapper_no_extract!(XrFrameState, xr::FrameState); xr_resource_wrapper!(XrViews, Vec); xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); xr_no_clone_resource_wrapper!(XrFrameWaiter, xr::FrameWaiter); +impl ExtractResource for XrFrameState { + type Source = Self; + + fn extract_resource(source: &Self::Source) -> Self { + let mut state = *source; + state.predicted_display_time = xr::Time::from_nanos( + state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(), + ); + state + } +} + pub(crate) struct VulkanOXrSessionSetupInfo { pub(crate) device_ptr: *const c_void, pub(crate) physical_device_ptr: *const c_void, diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index ac2e13b..ee88e4d 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -36,7 +36,7 @@ use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, }; -use self::xr_camera::{XrCameraType, TransformExtract}; +use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCameraType}; #[derive(Copy, Clone)] pub struct OpenXrInput { @@ -87,6 +87,7 @@ impl Plugin for OpenXrInput { app.add_plugins(ExtractComponentPlugin::::default()); app.add_plugins(ExtractComponentPlugin::::default()); app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); } } diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 912b3ec..50d38ed 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -7,7 +7,7 @@ use bevy::prelude::*; use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; use bevy::render::extract_component::ExtractComponent; use bevy::render::primitives::Frustum; -use bevy::render::view::{ColorGrading, VisibleEntities}; +use bevy::render::view::{ColorGrading, ExtractedView, VisibleEntities}; use openxr::Fovf; #[derive(Bundle)] @@ -79,6 +79,7 @@ impl ExtractComponent for GlobalTransformExtract { Some(*item) } } + #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum Eye { Left = 0, @@ -271,7 +272,7 @@ impl CameraProjection for XRProjection { } pub fn xr_camera_head_sync( - views: ResMut, + views: Res, mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, ) { //TODO calculate HMD position @@ -290,3 +291,25 @@ pub fn xr_camera_head_sync( transform.translation = view.pose.position.to_vec3(); } } +pub fn xr_camera_head_sync_render( + views: Res, + mut query: Query<(&mut ExtractedView, &XrCameraType)>, +) { + //TODO calculate HMD position + for (mut transform, camera_type) in query.iter_mut() { + // let mut t = Transform::IDENTITY; + // let view_idx = match camera_type { + // XrCameraType::Xr(eye) => *eye as usize, + // // I don't belive we need a flatscrenn cam, that's just a cam without this component + // XrCameraType::Flatscreen => continue, + // }; + // let view = match views.get(view_idx) { + // Some(views) => views, + // None => continue, + // }; + // t.rotation = view.pose.orientation.to_quat(); + // t.translation = view.pose.position.to_vec3(); + info!("cam update"); + transform.transform = GlobalTransform::IDENTITY; + } +} From 74c0813c487fcc9187fd0efe643470376be8ff0f Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 5 Feb 2024 02:43:03 +0100 Subject: [PATCH 15/25] wip --- src/lib.rs | 24 +----------------------- src/resource_macros.rs | 9 +++++++-- src/resources.rs | 28 ++++++++-------------------- 3 files changed, 16 insertions(+), 45 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 83b09f1..647b666 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -154,20 +154,6 @@ impl Plugin for OpenXrPlugin { .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), ); - // render_app.add_systems( - // Render, - // ( - // locate_views, - // xr_input::xr_camera::xr_camera_head_sync_render, - // // sync_simple_transforms, - // // propagate_transforms, - // // update_cam_views, - // ) - // .chain() - // .run_if(xr_only()) - // // .run_if(xr_render_only()) - // .in_set(RenderSet::Prepare), - // ); render_app.add_systems( Render, xr_end_frame @@ -187,14 +173,6 @@ impl Plugin for OpenXrPlugin { } } -// Confirmed Working -// Not Working Actually, the cam doesn't render with the new pose for some reason -fn update_cam_views(mut query: Query<(&mut ExtractedView, &GlobalTransform)>) { - for (mut view, transform) in &mut query { - view.transform = *transform; - } -} - fn xr_skip_frame( xr_swapchain: Res, xr_frame_state: Res, @@ -238,7 +216,7 @@ impl PluginGroup for DefaultXrPlugins { ..default() }, }) - // .disable::() + .disable::() .disable::() .add_before::(OpenXrPlugin { prefered_blend_mode: self.prefered_blend_mode, diff --git a/src/resource_macros.rs b/src/resource_macros.rs index bb83a63..11ffd20 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -33,10 +33,15 @@ macro_rules! xr_resource_wrapper { } #[macro_export] -macro_rules! xr_resource_wrapper_no_extract { +macro_rules! xr_resource_wrapper_copy { ($wrapper_type:ident, $xr_type:ty) => { #[derive( - Clone, Copy, bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut, + Clone, + Copy, + bevy::prelude::Resource, + bevy::prelude::Deref, + bevy::prelude::DerefMut, + bevy::render::extract_resource::ExtractResource, )] pub struct $wrapper_type($xr_type); diff --git a/src/resources.rs b/src/resources.rs index 2854abc..41678bb 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -4,36 +4,24 @@ use std::sync::Mutex; use crate::input::XrInput; // use crate::passthrough::XrPassthroughLayer; -use crate::xr_init::XrStatus; -use crate::{resource_macros::*, xr_resource_wrapper_no_extract}; +use crate::{resource_macros::*, xr_resource_wrapper_copy}; use bevy::prelude::*; -use bevy::render::extract_resource::{ExtractResource, ExtractResourcePlugin}; +use bevy::render::extract_resource::ExtractResourcePlugin; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); xr_resource_wrapper!(XrSession, xr::Session); -xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); -xr_resource_wrapper!(XrResolution, UVec2); -xr_resource_wrapper!(XrFormat, wgpu::TextureFormat); -xr_resource_wrapper_no_extract!(XrFrameState, xr::FrameState); +xr_resource_wrapper_copy!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); +xr_resource_wrapper_copy!(XrResolution, UVec2); +xr_resource_wrapper_copy!(XrFormat, wgpu::TextureFormat); +xr_resource_wrapper_copy!(XrFrameState, xr::FrameState); xr_resource_wrapper!(XrViews, Vec); xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); xr_no_clone_resource_wrapper!(XrFrameWaiter, xr::FrameWaiter); -impl ExtractResource for XrFrameState { - type Source = Self; - fn extract_resource(source: &Self::Source) -> Self { - let mut state = *source; - state.predicted_display_time = xr::Time::from_nanos( - state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(), - ); - state - } -} - -pub(crate) struct VulkanOXrSessionSetupInfo { +pub struct VulkanOXrSessionSetupInfo { pub(crate) device_ptr: *const c_void, pub(crate) physical_device_ptr: *const c_void, pub(crate) vk_instance_ptr: *const c_void, @@ -41,7 +29,7 @@ pub(crate) struct VulkanOXrSessionSetupInfo { pub(crate) xr_system_id: xr::SystemId, } -pub(crate) enum OXrSessionSetupInfo { +pub enum OXrSessionSetupInfo { Vulkan(VulkanOXrSessionSetupInfo), } From 3803968b7c7725a88fed4abb99dbc8cda910666e Mon Sep 17 00:00:00 2001 From: Schmarni Date: Mon, 12 Feb 2024 09:05:44 +0100 Subject: [PATCH 16/25] clean up passthrough, change hand entity behavior and add cleanup steps to some plugins --- examples/android/Cargo.toml | 8 +- examples/android/src/lib.rs | 36 +++++--- examples/demo/src/lib.rs | 4 +- examples/globe.rs | 3 +- examples/xr.rs | 3 +- src/lib.rs | 23 +++-- src/passthrough.rs | 61 ++++++------- src/xr_init/mod.rs | 3 +- src/xr_input/actions.rs | 18 +++- src/xr_input/hands/common.rs | 133 +++++++++++----------------- src/xr_input/hands/emulated.rs | 10 +-- src/xr_input/hands/hand_tracking.rs | 9 +- src/xr_input/hands/mod.rs | 62 +++++++------ 13 files changed, 178 insertions(+), 195 deletions(-) diff --git a/examples/android/Cargo.toml b/examples/android/Cargo.toml index bea6cb3..1c412bc 100644 --- a/examples/android/Cargo.toml +++ b/examples/android/Cargo.toml @@ -16,10 +16,10 @@ bevy_oxr.path = "../.." bevy = "0.12" openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = ["mint"] } -[profile.release] -lto = "fat" -codegen-units = 1 -panic = "abort" +# [profile.release] +# lto = "fat" +# codegen-units = 1 +# panic = "abort" # This metadata is used by `cargo-apk` - `xbuild` uses the `manifest.yaml` instead. [package.metadata.android] diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index efb835a..ba0f411 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -3,10 +3,10 @@ use bevy::prelude::*; use bevy::transform::components::Transform; use bevy_oxr::graphics::extensions::XrExtensions; use bevy_oxr::graphics::XrAppInfo; -use bevy_oxr::graphics::XrPreferdBlendMode::AlphaBlend; -use bevy_oxr::passthrough::{passthrough_layer_pause, passthrough_layer_resume}; -use bevy_oxr::xr_init::XrRenderData; +use bevy_oxr::passthrough::{PausePassthrough, ResumePassthrough, XrPassthroughState}; +use bevy_oxr::xr_init::xr_only; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; +use bevy_oxr::xr_input::hands::HandBone; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker, @@ -25,16 +25,23 @@ fn main() { }, prefered_blend_mode: bevy_oxr::graphics::XrPreferdBlendMode::Opaque, }) - .add_plugins(OpenXrDebugRenderer) + // .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) .add_systems(Startup, setup) - .add_systems(Update, (proto_locomotion, toggle_passthrough)) + .add_systems(Update, (proto_locomotion, toggle_passthrough).run_if(xr_only())) + .add_systems(Update, debug_hand_render.run_if(xr_only())) .add_systems(Startup, spawn_controllers_example) .insert_resource(PrototypeLocomotionConfig::default()) .run(); } +fn debug_hand_render(query: Query<&GlobalTransform, With>, mut gizmos: Gizmos) { + for transform in &query { + gizmos.sphere(transform.translation(), Quat::IDENTITY, 0.01, Color::RED); + } +} + /// set up a simple 3D scene fn setup( mut commands: Commands, @@ -90,15 +97,18 @@ fn spawn_controllers_example(mut commands: Commands) { )); } -// Does this work? Not getting logs -fn toggle_passthrough(keys: Res>, mut xr_data: ResMut) { +// TODO: make this a vr button +fn toggle_passthrough( + keys: Res>, + passthrough_state: Res, + mut resume: EventWriter, + mut pause: EventWriter, +) { if keys.just_pressed(KeyCode::Space) { - if xr_data.xr_passthrough_active { - passthrough_layer_pause(xr_data); - bevy::log::info!("Passthrough paused"); - } else { - passthrough_layer_resume(xr_data); - bevy::log::info!("Passthrough resumed"); + match *passthrough_state { + XrPassthroughState::Unsupported => {} + XrPassthroughState::Running => pause.send_default(), + XrPassthroughState::Paused => resume.send_default(), } } } diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index d55db1a..8f369f9 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -23,7 +23,7 @@ use bevy_oxr::{ xr_input::{ actions::XrActionSets, debug_gizmos::OpenXrDebugRenderer, - hands::common::{HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput}, + hands::common::{HandInputDebugRenderer, HandResource, HandsResource}, hands::HandBone, interactions::{ draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions, @@ -120,7 +120,7 @@ pub fn main() { //test capsule .add_systems(Startup, spawn_capsule) //physics hands - .add_plugins(OpenXrHandInput) + // .add_plugins(OpenXrHandInput) .add_plugins(HandInputDebugRenderer) .add_systems(Startup, spawn_physics_hands) .add_systems( diff --git a/examples/globe.rs b/examples/globe.rs index 192916b..3f439be 100644 --- a/examples/globe.rs +++ b/examples/globe.rs @@ -3,7 +3,7 @@ use bevy::prelude::*; use bevy::transform::components::Transform; use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::resources::XrViews; -use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput}; +use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer; use bevy_oxr::xr_input::interactions::{ InteractionEvent, XRDirectInteractor, XRInteractorState, XRRayInteractor, XRSocketInteractor, }; @@ -31,7 +31,6 @@ fn main() { .add_systems(Update, (proto_locomotion, pull_to_ground).chain()) .insert_resource(PrototypeLocomotionConfig::default()) .add_systems(Startup, spawn_controllers_example) - .add_plugins(OpenXrHandInput) .add_plugins(HandInputDebugRenderer) .add_event::() .run(); diff --git a/examples/xr.rs b/examples/xr.rs index 6f8d2de..68104d9 100644 --- a/examples/xr.rs +++ b/examples/xr.rs @@ -7,7 +7,7 @@ use bevy_oxr::input::XrInput; use bevy_oxr::resources::{XrFrameState, XrSession}; use bevy_oxr::xr_input::actions::XrActionSets; -use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput}; +use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer; use bevy_oxr::xr_input::interactions::{ draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions, update_interactable_states, InteractionEvent, Touched, XRDirectInteractor, XRInteractable, @@ -39,7 +39,6 @@ fn main() { .add_systems(Update, proto_locomotion) .insert_resource(PrototypeLocomotionConfig::default()) .add_systems(Startup, spawn_controllers_example) - .add_plugins(OpenXrHandInput) .add_plugins(HandInputDebugRenderer) .add_systems( Update, diff --git a/src/lib.rs b/src/lib.rs index ae0a5a4..17c5e6c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,11 +31,13 @@ use openxr as xr; use passthrough::{PassthroughPlugin, XrPassthroughLayer, XrPassthroughState}; use resources::*; use xr_init::{ - xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, XrEarlyInitPlugin, XrHasWaited, - XrShouldRender, XrStatus, + xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, SetupXrData, XrEarlyInitPlugin, + XrHasWaited, XrShouldRender, XrStatus, }; use xr_input::controllers::XrControllerType; -use xr_input::hands::XrHandPlugins; +use xr_input::hands::emulated::HandEmulationPlugin; +use xr_input::hands::hand_tracking::HandTrackingPlugin; +use xr_input::hands::HandPlugin; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -51,11 +53,6 @@ pub struct OpenXrPlugin { app_info: XrAppInfo, } -fn mr_test(mut commands: Commands, mut resume: EventWriter) { - commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); - resume.send_default(); -} - impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { app.insert_resource(XrSessionRunning::new(AtomicBool::new(false))); @@ -114,7 +111,6 @@ impl Plugin for OpenXrPlugin { app.add_plugins(RenderPlugin::default()); app.insert_resource(XrStatus::Disabled); } - app.add_systems(Update, mr_test.run_if(xr_only())); app.add_systems( PreUpdate, xr_poll_events.run_if(|status: Res| *status != XrStatus::NoInstance), @@ -146,8 +142,6 @@ impl Plugin for OpenXrPlugin { .run_if(xr_only()) .run_if(xr_after_wait_only()) .run_if(xr_render_only()) - // Do NOT touch this ordering! idk why but you can NOT just put in a RenderSet - // right before rendering .before(render_system) .after(RenderSet::ExtractCommands), // .in_set(RenderSet::Prepare), @@ -224,7 +218,9 @@ impl PluginGroup for DefaultXrPlugins { .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) .add_after::(XrInitPlugin) .add_before::(XrEarlyInitPlugin) - .add(XrHandPlugins) + .add(HandPlugin) + .add(HandTrackingPlugin) + .add(HandEmulationPlugin) .add(PassthroughPlugin) .add(XrResourcePlugin) .set(WindowPlugin { @@ -259,6 +255,7 @@ fn xr_poll_events( session: Option>, session_running: Res, mut app_exit: EventWriter, + mut setup_xr: EventWriter, mut cleanup_xr: EventWriter, ) { if let (Some(instance), Some(session)) = (instance, session) { @@ -274,6 +271,7 @@ fn xr_poll_events( xr::SessionState::READY => { info!("Calling Session begin :3"); session.begin(VIEW_TYPE).unwrap(); + setup_xr.send_default(); session_running.store(true, std::sync::atomic::Ordering::Relaxed); } xr::SessionState::STOPPING => { @@ -283,7 +281,6 @@ fn xr_poll_events( } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => { // app_exit.send(AppExit); - return; } _ => {} diff --git a/src/passthrough.rs b/src/passthrough.rs index 51256e7..e6eb1e5 100644 --- a/src/passthrough.rs +++ b/src/passthrough.rs @@ -25,10 +25,11 @@ pub enum XrPassthroughState { Paused, } -pub struct PassthroughPlugin; xr_arc_resource_wrapper!(XrPassthrough, xr::Passthrough); xr_arc_resource_wrapper!(XrPassthroughLayer, xr::PassthroughLayer); +pub struct PassthroughPlugin; + impl Plugin for PassthroughPlugin { fn build(&self, app: &mut App) { app.add_event::(); @@ -78,20 +79,31 @@ fn check_passthrough_support(mut cmds: Commands, instance: Option, mut state: ResMut) { +fn resume_passthrough( + layer: Res, + mut state: ResMut, + mut clear_color: ResMut, +) { if let Err(e) = layer.resume() { warn!("Unable to resume Passthrough: {}", e); return; } - info!("<=> Resume Passthrough"); + clear_color.set_a(0.0); + clear_color.set_r(0.0); + clear_color.set_g(0.0); + clear_color.set_b(0.0); *state = XrPassthroughState::Running; } -fn pause_passthrough(layer: Res, mut state: ResMut) { +fn pause_passthrough( + layer: Res, + mut state: ResMut, + mut clear_color: ResMut, +) { if let Err(e) = layer.pause() { warn!("Unable to resume Passthrough: {}", e); return; } - info!("<=> Pausing Passthrough"); + clear_color.set_a(1.0); *state = XrPassthroughState::Paused; } @@ -100,8 +112,8 @@ fn cleanup_passthrough(mut cmds: Commands) { cmds.remove_resource::(); } -fn setup_passthrough(mut cmds: Commands, instance: Res, session: Res) { - match create_passthrough(&instance, &session) { +fn setup_passthrough(mut cmds: Commands, session: Res) { + match create_passthrough(&session) { Ok((passthrough, layer)) => { cmds.insert_resource(XrPassthrough::from(passthrough)); cmds.insert_resource(XrPassthroughLayer::from(layer)); @@ -181,7 +193,6 @@ pub fn supports_passthrough(instance: &XrInstance, system: xr::SystemId) -> xr:: #[inline] pub fn create_passthrough( - instance: &XrInstance, xr_session: &XrSession, ) -> xr::Result<(xr::Passthrough, xr::PassthroughLayer)> { let passthrough = match xr_session { @@ -199,28 +210,14 @@ pub fn create_passthrough( Ok((passthrough, passthrough_layer)) } -// #[inline] -// pub fn passthrough_layer_resume(mut layer: ResMut, mut passthrough: ResMut) -> xr::Result<()> { -// layer.resume() -// } +/// Enable Passthrough on xr startup +/// just sends the [`ResumePassthrough`] event in [`XrSetup`] +pub struct EnablePassthroughStartup; -// #[inline] -// pub fn passthrough_layer_pause(mut xr_data_resource: ResMut) -> xr::Result<()> { -// unsafe { -// let passthrough_layer = &xr_data_resource.xr_passthrough_layer; -// { -// let passthrough_layer_locked = passthrough_layer.lock().unwrap(); -// cvt((xr_data_resource -// .xr_instance -// .exts() -// .fb_passthrough -// .unwrap() -// .passthrough_layer_pause)( -// *passthrough_layer_locked -// ))?; -// } -// xr_data_resource.xr_passthrough_active = false; -// bevy::log::info!("Paused passthrough layer"); -// Ok(()) -// } -// } +impl Plugin for EnablePassthroughStartup { + fn build(&self, app: &mut App) { + app.add_systems(XrSetup, |mut e: EventWriter| { + e.send_default(); + }); + } +} diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs index 06d9578..bd17a04 100644 --- a/src/xr_init/mod.rs +++ b/src/xr_init/mod.rs @@ -126,7 +126,7 @@ pub struct StartXrSession; pub struct EndXrSession; #[derive(Event, Clone, Copy, Default)] -struct SetupXrData; +pub(crate) struct SetupXrData; #[derive(Event, Clone, Copy, Default)] pub(crate) struct CleanupXrData; @@ -192,7 +192,6 @@ fn start_xr_session( commands.insert_resource(xr_views); commands.insert_resource(xr_frame_state); *status = XrStatus::Enabling; - setup_xr.send_default(); } fn stop_xr_session(session: ResMut, mut status: ResMut) { diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index 748968d..b9d8c2d 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -6,7 +6,7 @@ use xr::{Action, Binding, Haptic, Posef, Vector2f}; use crate::{ resources::{XrInstance, XrSession}, - xr_init::XrPrePostSetup, + xr_init::{XrCleanup, XrPrePostSetup, XrPreSetup}, }; use super::oculus_touch::ActionSets; @@ -16,13 +16,23 @@ pub use xr::sys::NULL_PATH; pub struct OpenXrActionsPlugin; impl Plugin for OpenXrActionsPlugin { fn build(&self, app: &mut App) { - app.insert_resource(SetupActionSets { - sets: HashMap::new(), - }); + app.add_systems(XrPreSetup, insert_setup_action_sets); app.add_systems(XrPrePostSetup, setup_oxr_actions); + app.add_systems(XrCleanup, clean_actions); } } +fn insert_setup_action_sets(mut cmds: Commands) { + cmds.insert_resource(SetupActionSets { + sets: HashMap::new(), + }); +} + +fn clean_actions(mut cmds: Commands) { + cmds.remove_resource::(); + cmds.remove_resource::(); +} + #[inline(always)] fn create_action( action: &SetupAction, diff --git a/src/xr_input/hands/common.rs b/src/xr_input/hands/common.rs index 104c6b3..f080613 100644 --- a/src/xr_input/hands/common.rs +++ b/src/xr_input/hands/common.rs @@ -1,6 +1,10 @@ -use bevy::prelude::{ - default, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, Plugin, PostUpdate, - Query, Resource, SpatialBundle, Startup, Transform, +use bevy::{ + core::Name, + prelude::{ + default, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, Plugin, PostUpdate, + Query, Resource, SpatialBundle, Startup, Transform, + }, + transform::components::GlobalTransform, }; use crate::xr_input::{trackers::OpenXRTracker, Hand}; @@ -8,14 +12,14 @@ use crate::xr_input::{trackers::OpenXRTracker, Hand}; use super::{BoneTrackingStatus, HandBone}; /// add debug renderer for controllers -#[derive(Default)] -pub struct OpenXrHandInput; - -impl Plugin for OpenXrHandInput { - fn build(&self, app: &mut bevy::prelude::App) { - app.add_systems(Startup, spawn_hand_entities); - } -} +// #[derive(Default)] +// pub struct OpenXrHandInput; +// +// impl Plugin for OpenXrHandInput { +// fn build(&self, app: &mut bevy::prelude::App) { +// app.add_systems(Startup, spawn_hand_entities); +// } +// } /// add debug renderer for controllers #[derive(Default)] @@ -161,75 +165,46 @@ pub fn spawn_hand_entities(mut commands: Commands) { for bone in bones.iter() { let boneid = commands .spawn(( + Name::new(format!("{:?} {:?}", hand, bone)), SpatialBundle::default(), - bone.clone(), + *bone, OpenXRTracker, - hand.clone(), + *hand, BoneTrackingStatus::Emulated, HandBoneRadius(0.1), )) .id(); - match hand { - Hand::Left => match bone { - HandBone::Palm => hand_resource.left.palm = boneid, - HandBone::Wrist => hand_resource.left.wrist = boneid, - HandBone::ThumbMetacarpal => hand_resource.left.thumb.metacarpal = boneid, - HandBone::ThumbProximal => hand_resource.left.thumb.proximal = boneid, - HandBone::ThumbDistal => hand_resource.left.thumb.distal = boneid, - HandBone::ThumbTip => hand_resource.left.thumb.tip = boneid, - HandBone::IndexMetacarpal => hand_resource.left.index.metacarpal = boneid, - HandBone::IndexProximal => hand_resource.left.index.proximal = boneid, - HandBone::IndexIntermediate => hand_resource.left.index.intermediate = boneid, - HandBone::IndexDistal => hand_resource.left.index.distal = boneid, - HandBone::IndexTip => hand_resource.left.index.tip = boneid, - HandBone::MiddleMetacarpal => hand_resource.left.middle.metacarpal = boneid, - HandBone::MiddleProximal => hand_resource.left.middle.proximal = boneid, - HandBone::MiddleIntermediate => hand_resource.left.middle.intermediate = boneid, - HandBone::MiddleDistal => hand_resource.left.middle.distal = boneid, - HandBone::MiddleTip => hand_resource.left.middle.tip = boneid, - HandBone::RingMetacarpal => hand_resource.left.ring.metacarpal = boneid, - HandBone::RingProximal => hand_resource.left.ring.proximal = boneid, - HandBone::RingIntermediate => hand_resource.left.ring.intermediate = boneid, - HandBone::RingDistal => hand_resource.left.ring.distal = boneid, - HandBone::RingTip => hand_resource.left.ring.tip = boneid, - HandBone::LittleMetacarpal => hand_resource.left.little.metacarpal = boneid, - HandBone::LittleProximal => hand_resource.left.little.proximal = boneid, - HandBone::LittleIntermediate => hand_resource.left.little.intermediate = boneid, - HandBone::LittleDistal => hand_resource.left.little.distal = boneid, - HandBone::LittleTip => hand_resource.left.little.tip = boneid, - }, - Hand::Right => match bone { - HandBone::Palm => hand_resource.right.palm = boneid, - HandBone::Wrist => hand_resource.right.wrist = boneid, - HandBone::ThumbMetacarpal => hand_resource.right.thumb.metacarpal = boneid, - HandBone::ThumbProximal => hand_resource.right.thumb.proximal = boneid, - HandBone::ThumbDistal => hand_resource.right.thumb.distal = boneid, - HandBone::ThumbTip => hand_resource.right.thumb.tip = boneid, - HandBone::IndexMetacarpal => hand_resource.right.index.metacarpal = boneid, - HandBone::IndexProximal => hand_resource.right.index.proximal = boneid, - HandBone::IndexIntermediate => hand_resource.right.index.intermediate = boneid, - HandBone::IndexDistal => hand_resource.right.index.distal = boneid, - HandBone::IndexTip => hand_resource.right.index.tip = boneid, - HandBone::MiddleMetacarpal => hand_resource.right.middle.metacarpal = boneid, - HandBone::MiddleProximal => hand_resource.right.middle.proximal = boneid, - HandBone::MiddleIntermediate => { - hand_resource.right.middle.intermediate = boneid - } - HandBone::MiddleDistal => hand_resource.right.middle.distal = boneid, - HandBone::MiddleTip => hand_resource.right.middle.tip = boneid, - HandBone::RingMetacarpal => hand_resource.right.ring.metacarpal = boneid, - HandBone::RingProximal => hand_resource.right.ring.proximal = boneid, - HandBone::RingIntermediate => hand_resource.right.ring.intermediate = boneid, - HandBone::RingDistal => hand_resource.right.ring.distal = boneid, - HandBone::RingTip => hand_resource.right.ring.tip = boneid, - HandBone::LittleMetacarpal => hand_resource.right.little.metacarpal = boneid, - HandBone::LittleProximal => hand_resource.right.little.proximal = boneid, - HandBone::LittleIntermediate => { - hand_resource.right.little.intermediate = boneid - } - HandBone::LittleDistal => hand_resource.right.little.distal = boneid, - HandBone::LittleTip => hand_resource.right.little.tip = boneid, - }, + let hand_res = match hand { + Hand::Left => &mut hand_resource.left, + Hand::Right => &mut hand_resource.right, + }; + match bone { + HandBone::Palm => hand_res.palm = boneid, + HandBone::Wrist => hand_res.wrist = boneid, + HandBone::ThumbMetacarpal => hand_res.thumb.metacarpal = boneid, + HandBone::ThumbProximal => hand_res.thumb.proximal = boneid, + HandBone::ThumbDistal => hand_res.thumb.distal = boneid, + HandBone::ThumbTip => hand_res.thumb.tip = boneid, + HandBone::IndexMetacarpal => hand_res.index.metacarpal = boneid, + HandBone::IndexProximal => hand_res.index.proximal = boneid, + HandBone::IndexIntermediate => hand_res.index.intermediate = boneid, + HandBone::IndexDistal => hand_res.index.distal = boneid, + HandBone::IndexTip => hand_res.index.tip = boneid, + HandBone::MiddleMetacarpal => hand_res.middle.metacarpal = boneid, + HandBone::MiddleProximal => hand_res.middle.proximal = boneid, + HandBone::MiddleIntermediate => hand_res.middle.intermediate = boneid, + HandBone::MiddleDistal => hand_res.middle.distal = boneid, + HandBone::MiddleTip => hand_res.middle.tip = boneid, + HandBone::RingMetacarpal => hand_res.ring.metacarpal = boneid, + HandBone::RingProximal => hand_res.ring.proximal = boneid, + HandBone::RingIntermediate => hand_res.ring.intermediate = boneid, + HandBone::RingDistal => hand_res.ring.distal = boneid, + HandBone::RingTip => hand_res.ring.tip = boneid, + HandBone::LittleMetacarpal => hand_res.little.metacarpal = boneid, + HandBone::LittleProximal => hand_res.little.proximal = boneid, + HandBone::LittleIntermediate => hand_res.little.intermediate = boneid, + HandBone::LittleDistal => hand_res.little.distal = boneid, + HandBone::LittleTip => hand_res.little.tip = boneid, } } } @@ -241,16 +216,12 @@ pub struct HandBoneRadius(pub f32); pub fn draw_hand_entities( mut gizmos: Gizmos, - query: Query<(&Transform, &HandBone, &HandBoneRadius)>, + query: Query<(&GlobalTransform, &HandBone, &HandBoneRadius)>, ) { for (transform, hand_bone, hand_bone_radius) in query.iter() { let (_, color) = get_bone_gizmo_style(hand_bone); - gizmos.sphere( - transform.translation, - transform.rotation, - hand_bone_radius.0, - color, - ); + let (_, rotation, translation) = transform.to_scale_rotation_translation(); + gizmos.sphere(translation, rotation, hand_bone_radius.0, color); } } diff --git a/src/xr_input/hands/emulated.rs b/src/xr_input/hands/emulated.rs index ab1e627..9ebbc73 100644 --- a/src/xr_input/hands/emulated.rs +++ b/src/xr_input/hands/emulated.rs @@ -92,8 +92,6 @@ fn setup_hand_emulation_action_set(mut action_sets: ResMut) { suggest_oculus_touch_profile(action_set); } -pub struct EmulatedHandPoseData {} - fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) { action_set.suggest_binding( "/interaction_profiles/oculus/touch_controller", @@ -131,7 +129,6 @@ pub(crate) fn update_hand_skeleton_from_emulated( action_sets: Res, left_controller_transform: Query<&Transform, With>, right_controller_transform: Query<&Transform, With>, - tracking_root_transform: Query<&Transform, With>, mut bones: Query< ( &mut Transform, @@ -226,7 +223,6 @@ pub(crate) fn update_hand_skeleton_from_emulated( }, } } - let trt = tracking_root_transform.single(); for (mut t, bone, hand, status, mut radius) in bones.iter_mut() { match status { BoneTrackingStatus::Emulated => {} @@ -238,9 +234,9 @@ pub(crate) fn update_hand_skeleton_from_emulated( Hand::Left => 0, Hand::Right => 1, }][bone.get_index_from_bone()]; - *t = t.with_scale(trt.scale); - *t = t.with_rotation(trt.rotation * t.rotation); - *t = t.with_translation(trt.transform_point(t.translation)); + // *t = t.with_scale(trt.scale); + // *t = t.with_rotation(trt.rotation * t.rotation); + // *t = t.with_translation(trt.transform_point(t.translation)); } } pub fn update_hand_bones_emulated( diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index fd8c920..e2326a1 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -6,7 +6,7 @@ use crate::{ input::XrInput, resources::{XrFrameState, XrSession}, xr_init::xr_only, - xr_input::{hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv, Vec3Conv}, + xr_input::{hands::HandBone, Hand, QuatConv, Vec3Conv}, }; use super::BoneTrackingStatus; @@ -158,7 +158,6 @@ pub fn update_hand_bones( hand_tracking: Option>, xr_input: Res, xr_frame_state: Res, - root_query: Query<(&Transform, With, Without)>, mut bones: Query<( &mut Transform, &Hand, @@ -174,7 +173,6 @@ pub fn update_hand_bones( return; } }; - let (root_transform, _, _) = root_query.get_single().unwrap(); let left_hand_data = hand_ref.get_poses(Hand::Left); let right_hand_data = hand_ref.get_poses(Hand::Right); bones @@ -203,8 +201,7 @@ pub fn update_hand_bones( *status = BoneTrackingStatus::Tracked; } radius.0 = bone_data.radius; - *transform = transform - .with_translation(root_transform.transform_point(bone_data.position)) - .with_rotation(root_transform.rotation * bone_data.orientation) + transform.translation = bone_data.position; + transform.rotation = bone_data.orientation; }); } diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs index fa85cf8..8ab17d8 100644 --- a/src/xr_input/hands/mod.rs +++ b/src/xr_input/hands/mod.rs @@ -1,38 +1,50 @@ -use bevy::{app::PluginGroupBuilder, prelude::*}; +use bevy::prelude::*; use openxr::FormFactor; use crate::{ resources::{XrInstance, XrSession}, - xr_init::XrPreSetup, + xr_init::{XrCleanup, XrPreSetup, XrSetup}, }; use self::{ - emulated::HandEmulationPlugin, - hand_tracking::{DisableHandTracking, HandTrackingData, HandTrackingPlugin}, + common::{spawn_hand_entities, HandBoneRadius, HandsResource}, + hand_tracking::{DisableHandTracking, HandTrackingData}, }; +use super::{trackers::OpenXRTracker, Hand}; + pub mod common; pub mod emulated; pub mod hand_tracking; -pub struct XrHandPlugins; - -impl Plugin for XrHandPlugins { - fn build(&self, app: &mut App) { - app.add_plugins(HandTrackingPlugin) - .add_plugins(HandPlugin) - .add_plugins(HandEmulationPlugin); - } -} - pub struct HandPlugin; impl Plugin for HandPlugin { fn build(&self, app: &mut App) { app.add_systems(XrPreSetup, check_for_handtracking); + app.add_systems(XrSetup, spawn_hand_entities); + app.add_systems(XrCleanup, despawn_hand_entities); } } +#[allow(clippy::type_complexity)] +fn despawn_hand_entities( + mut commands: Commands, + hand_entities: Query< + Entity, + ( + With, + With, + With, + ), + >, +) { + for e in &hand_entities { + commands.entity(e).despawn_recursive(); + } + commands.remove_resource::() +} + fn check_for_handtracking( mut commands: Commands, instance: Res, @@ -86,21 +98,17 @@ pub enum HandBone { } impl HandBone { pub fn is_finger(&self) -> bool { - match &self { - HandBone::Wrist => false, - HandBone::Palm => false, - _ => true, - } + !matches!(self, HandBone::Wrist | HandBone::Palm) } pub fn is_metacarpal(&self) -> bool { - match &self { - HandBone::ThumbMetacarpal => true, - HandBone::IndexMetacarpal => true, - HandBone::MiddleMetacarpal => true, - HandBone::RingMetacarpal => true, - HandBone::LittleTip => true, - _ => false, - } + matches!( + self, + HandBone::ThumbMetacarpal + | HandBone::IndexMetacarpal + | HandBone::MiddleMetacarpal + | HandBone::RingMetacarpal + | HandBone::LittleTip + ) } pub const fn get_all_bones() -> [HandBone; 26] { [ From 31d7b05b4a288aea3e3abddfe476b14f9c02d85f Mon Sep 17 00:00:00 2001 From: Schmarni Date: Thu, 15 Feb 2024 06:04:16 +0100 Subject: [PATCH 17/25] handtracking not working on quest 2 v62 --- examples/android/src/lib.rs | 9 ++- src/lib.rs | 17 +++-- src/resources.rs | 1 - src/xr_init/mod.rs | 4 +- src/xr_input/actions.rs | 41 +++++++++-- src/xr_input/hands/common.rs | 2 +- src/xr_input/hands/hand_tracking.rs | 12 +++- src/xr_input/mod.rs | 93 ++++++++---------------- src/xr_input/xr_camera.rs | 105 ++++++++++++++++++---------- 9 files changed, 163 insertions(+), 121 deletions(-) diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index ba0f411..a52a93c 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -6,6 +6,7 @@ use bevy_oxr::graphics::XrAppInfo; use bevy_oxr::passthrough::{PausePassthrough, ResumePassthrough, XrPassthroughState}; use bevy_oxr::xr_init::xr_only; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; +use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer; use bevy_oxr::xr_input::hands::HandBone; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ @@ -17,6 +18,7 @@ use bevy_oxr::DefaultXrPlugins; fn main() { let mut xr_extensions = XrExtensions::default(); xr_extensions.enable_fb_passthrough(); + xr_extensions.enable_hand_tracking(); App::new() .add_plugins(DefaultXrPlugins { reqeusted_extensions: xr_extensions, @@ -28,8 +30,13 @@ fn main() { // .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) + .add_plugins(HandInputDebugRenderer) + .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup) .add_systems(Startup, setup) - .add_systems(Update, (proto_locomotion, toggle_passthrough).run_if(xr_only())) + .add_systems( + Update, + (proto_locomotion, toggle_passthrough).run_if(xr_only()), + ) .add_systems(Update, debug_hand_render.run_if(xr_only())) .add_systems(Startup, spawn_controllers_example) .insert_resource(PrototypeLocomotionConfig::default()) diff --git a/src/lib.rs b/src/lib.rs index 17c5e6c..49c8508 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,12 +6,8 @@ pub mod resources; pub mod xr_init; pub mod xr_input; -use std::fs::File; -use std::io::{BufWriter, Write}; use std::sync::atomic::AtomicBool; -use std::sync::{Arc, Mutex}; -use crate::passthrough::ResumePassthrough; use crate::xr_init::{StartXrSession, XrInitPlugin}; use crate::xr_input::oculus_touch::ActionSets; use bevy::app::{AppExit, PluginGroupBuilder}; @@ -34,10 +30,12 @@ use xr_init::{ xr_after_wait_only, xr_only, xr_render_only, CleanupXrData, SetupXrData, XrEarlyInitPlugin, XrHasWaited, XrShouldRender, XrStatus, }; +use xr_input::actions::OpenXrActionsPlugin; use xr_input::controllers::XrControllerType; use xr_input::hands::emulated::HandEmulationPlugin; use xr_input::hands::hand_tracking::HandTrackingPlugin; use xr_input::hands::HandPlugin; +use xr_input::xr_camera::XrCameraPlugin; use xr_input::OpenXrInput; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; @@ -81,7 +79,7 @@ impl Plugin for OpenXrPlugin { debug!("Configured wgpu adapter Features: {:#?}", device.features()); warn!("Starting with OpenXR Instance"); app.insert_resource(xr_instance.clone()); - app.insert_resource(blend_mode.clone()); + app.insert_resource(blend_mode); app.insert_resource(ActionSets(vec![])); app.insert_resource(xr_instance); app.insert_resource(blend_mode); @@ -120,7 +118,9 @@ impl Plugin for OpenXrPlugin { ( xr_reset_per_frame_resources, xr_wait_frame.run_if(xr_only()), + // xr_begin_frame.run_if(xr_only()), locate_views.run_if(xr_only()), + apply_deferred, ) .chain() @@ -215,8 +215,10 @@ impl PluginGroup for DefaultXrPlugins { reqeusted_extensions: self.reqeusted_extensions, app_info: self.app_info.clone(), }) - .add_after::(OpenXrInput::new(XrControllerType::OculusTouch)) - .add_after::(XrInitPlugin) + .add(XrInitPlugin) + .add(OpenXrInput::new(XrControllerType::OculusTouch)) + .add(OpenXrActionsPlugin) + .add(XrCameraPlugin) .add_before::(XrEarlyInitPlugin) .add(HandPlugin) .add(HandTrackingPlugin) @@ -318,6 +320,7 @@ pub fn xr_wait_frame( return; } }; + // frame_state.predicted_display_time = xr::Time::from_nanos(frame_state.predicted_display_time.as_nanos() + frame_state.predicted_display_period.as_nanos()); **should_render = frame_state.should_render; **waited = true; } diff --git a/src/resources.rs b/src/resources.rs index 3b6ee26..305c76a 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -16,7 +16,6 @@ use core::ptr; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); -// xr_resource_wrapper!(XrSession, xr::Session); xr_resource_wrapper_copy!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode); xr_resource_wrapper_copy!(XrResolution, UVec2); xr_resource_wrapper_copy!(XrFormat, wgpu::TextureFormat); diff --git a/src/xr_init/mod.rs b/src/xr_init/mod.rs index bd17a04..70ac9c9 100644 --- a/src/xr_init/mod.rs +++ b/src/xr_init/mod.rs @@ -54,6 +54,7 @@ pub fn xr_after_wait_only() -> impl FnMut(Res) -> bool { impl Plugin for XrEarlyInitPlugin { fn build(&self, app: &mut App) { + add_schedules(app); app.add_event::() .add_event::() .add_event::() @@ -63,7 +64,6 @@ impl Plugin for XrEarlyInitPlugin { impl Plugin for XrInitPlugin { fn build(&self, app: &mut App) { - add_schedules(app); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); app.add_plugins(ExtractResourcePlugin::::default()); @@ -106,7 +106,9 @@ fn setup_manual_texture_views( } pub fn setup_xr(world: &mut World) { + info!("Pre XrPreSetup"); world.run_schedule(XrPreSetup); + info!("Post XrPreSetup"); world.run_schedule(XrSetup); world.run_schedule(XrPrePostSetup); world.run_schedule(XrPostSetup); diff --git a/src/xr_input/actions.rs b/src/xr_input/actions.rs index b9d8c2d..940f40a 100644 --- a/src/xr_input/actions.rs +++ b/src/xr_input/actions.rs @@ -6,7 +6,7 @@ use xr::{Action, Binding, Haptic, Posef, Vector2f}; use crate::{ resources::{XrInstance, XrSession}, - xr_init::{XrCleanup, XrPrePostSetup, XrPreSetup}, + xr_init::{xr_only, XrCleanup, XrPrePostSetup, XrPreSetup}, }; use super::oculus_touch::ActionSets; @@ -16,13 +16,18 @@ pub use xr::sys::NULL_PATH; pub struct OpenXrActionsPlugin; impl Plugin for OpenXrActionsPlugin { fn build(&self, app: &mut App) { - app.add_systems(XrPreSetup, insert_setup_action_sets); + app.add_systems(PreUpdate, sync_actions.run_if(xr_only())); + app.add_systems( + XrPreSetup, + (insert_setup_action_sets, apply_deferred).chain(), + ); app.add_systems(XrPrePostSetup, setup_oxr_actions); app.add_systems(XrCleanup, clean_actions); } } fn insert_setup_action_sets(mut cmds: Commands) { + info!("WHAT?!"); cmds.insert_resource(SetupActionSets { sets: HashMap::new(), }); @@ -57,7 +62,6 @@ pub fn setup_oxr_actions(world: &mut World) { let right_path = instance.string_to_path("/user/hand/right").unwrap(); let hands = [left_path, right_path]; - let mut oxr_action_sets = Vec::new(); let mut action_sets = XrActionSets { sets: default() }; // let mut action_bindings: HashMap<&'static str, Vec> = HashMap::new(); let mut action_bindings: HashMap< @@ -101,11 +105,11 @@ pub fn setup_oxr_actions(world: &mut World) { } } } - oxr_action_sets.push(oxr_action_set); + // oxr_action_sets.push(oxr_action_set); action_sets.sets.insert( set_name, ActionSet { - // oxr_action_set, + oxr_action_set, actions, enabled: true, }, @@ -152,10 +156,15 @@ pub fn setup_oxr_actions(world: &mut World) { .expect("Unable to suggest interaction bindings!"); } session - .attach_action_sets(&oxr_action_sets.iter().collect::>()) + .attach_action_sets( + &action_sets + .sets + .values() + .map(|set| &set.oxr_action_set) + .collect::>(), + ) .expect("Unable to attach action sets!"); - world.insert_resource(ActionSets(oxr_action_sets)); world.insert_resource(action_sets); } @@ -267,6 +276,7 @@ pub struct ActionSet { // add functionality to enable/disable action sets enabled: bool, actions: HashMap<&'static str, TypedAction>, + oxr_action_set: xr::ActionSet, } #[derive(Resource)] @@ -380,3 +390,20 @@ impl XrActionSets { } } } + +pub fn sync_actions(action_sets: Res, session: Res) { + let active_sets = action_sets + .sets + .values() + .filter_map(|set| { + if set.enabled { + Some(xr::ActiveActionSet::new(&set.oxr_action_set)) + } else { + None + } + }) + .collect::>(); + if let Err(err) = session.sync_actions(&active_sets) { + warn!("OpenXR action sync error: {}", err); + } +} diff --git a/src/xr_input/hands/common.rs b/src/xr_input/hands/common.rs index f080613..7958e10 100644 --- a/src/xr_input/hands/common.rs +++ b/src/xr_input/hands/common.rs @@ -170,7 +170,7 @@ pub fn spawn_hand_entities(mut commands: Commands) { *bone, OpenXRTracker, *hand, - BoneTrackingStatus::Emulated, + BoneTrackingStatus::Tracked, HandBoneRadius(0.1), )) .id(); diff --git a/src/xr_input/hands/hand_tracking.rs b/src/xr_input/hands/hand_tracking.rs index e2326a1..0c632f8 100644 --- a/src/xr_input/hands/hand_tracking.rs +++ b/src/xr_input/hands/hand_tracking.rs @@ -63,6 +63,7 @@ pub struct HandJoint { pub radius: f32, } +#[derive(Debug)] pub struct HandJoints { inner: [HandJoint; 26], } @@ -175,9 +176,15 @@ pub fn update_hand_bones( }; let left_hand_data = hand_ref.get_poses(Hand::Left); let right_hand_data = hand_ref.get_poses(Hand::Right); + if left_hand_data.is_none() || right_hand_data.is_none() { + error!("something is very wrong for hand_tracking!! doesn't have data for both hands!"); + } + + info!("hand_tracking"); bones .par_iter_mut() .for_each(|(mut transform, hand, bone, mut radius, mut status)| { + info!("hand_tracking bone before filter"); match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) { (Hand::Left, Some(DisableHandTracking::OnlyLeft)) => { *status = BoneTrackingStatus::Emulated; @@ -189,14 +196,17 @@ pub fn update_hand_bones( } _ => {} } + info!("hand_tracking bone mid filter"); let bone_data = match (hand, &left_hand_data, &right_hand_data) { (Hand::Left, Some(data), _) => data.get_joint(*bone), (Hand::Right, _, Some(data)) => data.get_joint(*bone), - _ => { + (hand, left_data, right_data) => { + info!("{:?},{:?},{:?}", hand, left_data, right_data); *status = BoneTrackingStatus::Emulated; return; } }; + info!("hand_tracking bone after filter"); if *status == BoneTrackingStatus::Emulated { *status = BoneTrackingStatus::Tracked; } diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs index ee88e4d..13c96df 100644 --- a/src/xr_input/mod.rs +++ b/src/xr_input/mod.rs @@ -10,7 +10,7 @@ pub mod trackers; pub mod xr_camera; use crate::resources::{XrInstance, XrSession}; -use crate::xr_init::{xr_only, XrPostSetup, XrPreSetup, XrSetup}; +use crate::xr_init::{xr_only, XrCleanup, XrPostSetup, XrPreSetup, XrSetup}; use crate::xr_input::controllers::XrControllerType; use crate::xr_input::oculus_touch::setup_oculus_controller; use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle}; @@ -19,6 +19,7 @@ use bevy::app::{App, PostUpdate, Startup}; use bevy::ecs::entity::Entity; use bevy::ecs::query::With; use bevy::ecs::system::Query; +use bevy::hierarchy::DespawnRecursiveExt; use bevy::log::{info, warn}; use bevy::math::Vec2; use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource}; @@ -36,7 +37,7 @@ use self::trackers::{ adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye, OpenXRTrackingRoot, }; -use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCameraType}; +use self::xr_camera::{GlobalTransformExtract, TransformExtract, XrCamera}; #[derive(Copy, Clone)] pub struct OpenXrInput { @@ -56,9 +57,8 @@ impl OpenXrInput { impl Plugin for OpenXrInput { fn build(&self, app: &mut App) { - app.add_plugins(CameraProjectionPlugin::::default()); - app.add_plugins(OpenXrActionsPlugin); app.add_systems(XrPostSetup, post_action_setup_oculus_controller); + // why only when the controller is oculus? that is still backed by generic actions match self.controller_type { XrControllerType::OculusTouch => { app.add_systems(XrSetup, setup_oculus_controller); @@ -66,77 +66,42 @@ impl Plugin for OpenXrInput { } //adopt any new trackers app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only())); - app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); - app.add_systems( - PreUpdate, - xr_camera_head_sync - .run_if(xr_only()) - .after(xr_wait_frame) - .after(locate_views), - ); + // app.add_systems(PreUpdate, action_set_system.run_if(xr_only())); //update controller trackers app.add_systems(Update, update_open_xr_controllers.run_if(xr_only())); - app.add_systems( - PostUpdate, - update_frusta:: - .after(TransformSystem::TransformPropagate) - .before(VisibilitySystems::UpdatePerspectiveFrusta), - ); app.add_systems(XrPreSetup, init_subaction_path); - app.add_systems(XrSetup, setup_xr_cameras); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); - app.add_plugins(ExtractComponentPlugin::::default()); + app.add_systems(XrSetup, setup_xr_root); + app.add_systems(XrCleanup, cleanup_xr_root); } } -#[derive(Deref, DerefMut, Resource)] -pub struct InteractionProfileBindings(pub HashMap<&'static str, Vec>>); - -fn setup_binding_recommendations( - mut commands: Commands, - instance: Res, - bindings: Res, -) { - commands.remove_resource::(); -} - -fn setup_xr_cameras( +fn cleanup_xr_root( mut commands: Commands, tracking_root_query: Query>, ) { - //this needs to do the whole xr tracking volume not just cameras - //get the root? - - let tracking_root = match tracking_root_query.get_single() { - Ok(e) => e, - Err(_) => commands - .spawn((SpatialBundle::default(), OpenXRTrackingRoot)) - .id(), - }; - let right = commands - .spawn((XrCameraBundle::new(Eye::Right), OpenXRRightEye)) - .id(); - let left = commands - .spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye)) - .id(); - commands.entity(tracking_root).push_children(&[right, left]); -} - -pub fn action_set_system(action_sets: Res, session: Res) { - let mut active_action_sets = vec![]; - for i in &action_sets.0 { - active_action_sets.push(openxr::ActiveActionSet::new(i)); - } - //info!("action sets: {:#?}", action_sets.0.len()); - match session.sync_actions(&active_action_sets) { - Err(err) => { - warn!("{}", err); - } - _ => {} + for e in &tracking_root_query { + commands.entity(e).despawn_recursive(); } } +fn setup_xr_root( + mut commands: Commands, + tracking_root_query: Query>, +) { + if tracking_root_query.get_single().is_err() { + commands.spawn((SpatialBundle::default(), OpenXRTrackingRoot)); + } +} + +// pub fn action_set_system(action_sets: Res, session: Res) { +// let mut active_action_sets = vec![]; +// for i in &action_sets.0 { +// active_action_sets.push(openxr::ActiveActionSet::new(i)); +// } +// //info!("action sets: {:#?}", action_sets.0.len()); +// if let Err(err) = session.sync_actions(&active_action_sets) { +// warn!("{}", err); +// } +// } pub trait Vec2Conv { fn to_vec2(&self) -> Vec2; diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs index 50d38ed..f154d4c 100644 --- a/src/xr_input/xr_camera.rs +++ b/src/xr_input/xr_camera.rs @@ -1,15 +1,73 @@ +use crate::xr_init::{xr_only, XrCleanup, XrSetup}; use crate::xr_input::{QuatConv, Vec3Conv}; -use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; +use crate::{locate_views, xr_wait_frame, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; use bevy::ecs::system::lifetimeless::Read; use bevy::math::Vec3A; use bevy::prelude::*; -use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; -use bevy::render::extract_component::ExtractComponent; +use bevy::render::camera::{ + CameraProjection, CameraProjectionPlugin, CameraRenderGraph, RenderTarget, +}; +use bevy::render::extract_component::{ExtractComponent, ExtractComponentPlugin}; use bevy::render::primitives::Frustum; -use bevy::render::view::{ColorGrading, ExtractedView, VisibleEntities}; +use bevy::render::view::{ + update_frusta, ColorGrading, ExtractedView, VisibilitySystems, VisibleEntities, +}; +use bevy::transform::TransformSystem; use openxr::Fovf; +use super::trackers::{OpenXRLeftEye, OpenXRRightEye, OpenXRTracker, OpenXRTrackingRoot}; + +pub struct XrCameraPlugin; + +impl Plugin for XrCameraPlugin { + fn build(&self, app: &mut App) { + app.add_plugins(CameraProjectionPlugin::::default()); + app.add_systems( + PreUpdate, + xr_camera_head_sync + .run_if(xr_only()) + .after(xr_wait_frame) + .after(locate_views), + ); + // a little late latching + app.add_systems( + PostUpdate, + xr_camera_head_sync + .before(TransformSystem::TransformPropagate) + .run_if(xr_only()), + ); + app.add_systems( + PostUpdate, + update_frusta:: + .after(TransformSystem::TransformPropagate) + .before(VisibilitySystems::UpdatePerspectiveFrusta), + ); + app.add_systems(XrSetup, setup_xr_cameras); + app.add_systems(XrCleanup, cleanup_xr_cameras); + app.add_plugins(ExtractComponentPlugin::::default()); + app.add_plugins(ExtractComponentPlugin::::default()); + // app.add_plugins(ExtractComponentPlugin::::default()); + // app.add_plugins(ExtractComponentPlugin::::default()); + } +} + +// might be unnesesary since it should be parented to the root +fn cleanup_xr_cameras(mut commands: Commands, entities: Query>) { + for e in &entities { + commands.entity(e).despawn_recursive(); + } +} + +fn setup_xr_cameras(mut commands: Commands) { + commands.spawn(( + XrCameraBundle::new(Eye::Right), + OpenXRRightEye, + OpenXRTracker, + )); + commands.spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye, OpenXRTracker)); +} + #[derive(Bundle)] pub struct XrCamerasBundle { pub left: XrCameraBundle, @@ -42,13 +100,10 @@ pub struct XrCameraBundle { pub tonemapping: Tonemapping, pub dither: DebandDither, pub color_grading: ColorGrading, - pub xr_camera_type: XrCameraType, + pub xr_camera_type: XrCamera, } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)] -pub enum XrCameraType { - Xr(Eye), - Flatscreen, -} +pub struct XrCamera(Eye); #[derive(Component)] pub(super) struct TransformExtract; @@ -108,7 +163,7 @@ impl XrCameraBundle { tonemapping: Default::default(), dither: DebandDither::Enabled, color_grading: Default::default(), - xr_camera_type: XrCameraType::Xr(eye), + xr_camera_type: XrCamera(eye), } } } @@ -273,15 +328,11 @@ impl CameraProjection for XRProjection { pub fn xr_camera_head_sync( views: Res, - mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, + mut query: Query<(&mut Transform, &XrCamera, &mut XRProjection)>, ) { //TODO calculate HMD position for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - XrCameraType::Xr(eye) => *eye as usize, - // I don't belive we need a flatscrenn cam, that's just a cam without this component - XrCameraType::Flatscreen => continue, - }; + let view_idx = camera_type.0 as usize; let view = match views.get(view_idx) { Some(views) => views, None => continue, @@ -291,25 +342,3 @@ pub fn xr_camera_head_sync( transform.translation = view.pose.position.to_vec3(); } } -pub fn xr_camera_head_sync_render( - views: Res, - mut query: Query<(&mut ExtractedView, &XrCameraType)>, -) { - //TODO calculate HMD position - for (mut transform, camera_type) in query.iter_mut() { - // let mut t = Transform::IDENTITY; - // let view_idx = match camera_type { - // XrCameraType::Xr(eye) => *eye as usize, - // // I don't belive we need a flatscrenn cam, that's just a cam without this component - // XrCameraType::Flatscreen => continue, - // }; - // let view = match views.get(view_idx) { - // Some(views) => views, - // None => continue, - // }; - // t.rotation = view.pose.orientation.to_quat(); - // t.translation = view.pose.position.to_vec3(); - info!("cam update"); - transform.transform = GlobalTransform::IDENTITY; - } -} From a95462b91d095e3e72a47de34395c0300d7e7625 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 20 Feb 2024 06:57:06 +0100 Subject: [PATCH 18/25] pre bevy 0.13 --- src/lib.rs | 19 +++++++++++-------- src/xr_input/hands/mod.rs | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 49c8508..9a3fb3b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -120,7 +120,6 @@ impl Plugin for OpenXrPlugin { xr_wait_frame.run_if(xr_only()), // xr_begin_frame.run_if(xr_only()), locate_views.run_if(xr_only()), - apply_deferred, ) .chain() @@ -306,23 +305,27 @@ fn xr_begin_frame(swapchain: Res) { } pub fn xr_wait_frame( - mut frame_state: ResMut, - mut frame_waiter: ResMut, - mut should_render: ResMut, - mut waited: ResMut, + world: &mut World, + // mut frame_state: ResMut, + // mut frame_waiter: ResMut, + // mut should_render: ResMut, + // mut waited: ResMut, ) { + let mut frame_waiter = world.get_resource_mut::().unwrap(); { let _span = info_span!("xr_wait_frame").entered(); - *frame_state = match frame_waiter.wait() { + + *world.get_resource_mut::().unwrap() = match frame_waiter.wait() { Ok(a) => a.into(), Err(e) => { warn!("error: {}", e); return; } }; + let should_render = world.get_resource::().unwrap().should_render; // frame_state.predicted_display_time = xr::Time::from_nanos(frame_state.predicted_display_time.as_nanos() + frame_state.predicted_display_period.as_nanos()); - **should_render = frame_state.should_render; - **waited = true; + **world.get_resource_mut::().unwrap() = should_render; + **world.get_resource_mut::().unwrap() = true; } } diff --git a/src/xr_input/hands/mod.rs b/src/xr_input/hands/mod.rs index 8ab17d8..ea7a230 100644 --- a/src/xr_input/hands/mod.rs +++ b/src/xr_input/hands/mod.rs @@ -55,6 +55,7 @@ fn check_for_handtracking( .supports_hand_tracking(instance.system(FormFactor::HEAD_MOUNTED_DISPLAY).unwrap()) .is_ok_and(|v| v); if hands { + info!("handtracking!"); commands.insert_resource(HandTrackingData::new(&session).unwrap()); } else { commands.insert_resource(DisableHandTracking::Both); From df71c7931d54c9fa5e8b9e1e201c2c3b5f9171fc Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 20 Feb 2024 09:15:21 +0100 Subject: [PATCH 19/25] update to bevy 0.13. TODO: fix view weirdness and do a pass over most of xr_input to turn the modules into plugins --- Cargo.toml | 12 ++--- examples/android/Cargo.toml | 2 +- examples/android/src/lib.rs | 26 ++++++---- examples/demo/Cargo.toml | 5 +- examples/demo/src/lib.rs | 55 ++++++++------------ examples/demo/src/setup.rs | 17 +++---- examples/globe.rs | 16 +++--- examples/xr.rs | 25 +++++----- src/graphics/vulkan.rs | 13 +++-- src/lib.rs | 2 + src/xr_input/debug_gizmos.rs | 23 +++++---- src/xr_input/interactions.rs | 17 ++++--- src/xr_input/mod.rs | 2 +- src/xr_input/prototype_locomotion.rs | 30 ++++++----- src/xr_input/trackers.rs | 36 +++++++------ src/xr_input/xr_camera.rs | 75 +++++++++++++++------------- 16 files changed, 179 insertions(+), 177 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bb93ce7..847c1b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,12 +17,12 @@ members = ["examples/android", "examples/demo"] [dependencies] # anyhow = "1.0.75" ash = "0.37.3" -bevy = "0.12" +bevy = "0.13" futures-lite = "2.0.1" mint = "0.5.9" -wgpu = "0.17.1" -wgpu-core = { version = "0.17.1", features = ["vulkan"] } -wgpu-hal = "0.17.1" +wgpu = "0.19" +wgpu-core = { version = "0.19", features = ["vulkan"] } +wgpu-hal = "0.19" eyre = "0.6.11" [target.'cfg(windows)'.dependencies] @@ -46,7 +46,7 @@ ndk-context = "0.1" jni = "0.20" [dev-dependencies] -bevy = "0.12" +bevy = "0.13" bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } color-eyre = "0.6.2" @@ -58,4 +58,4 @@ path = "examples/xr.rs" debug = true [patch.crates-io] -ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } +# ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" } diff --git a/examples/android/Cargo.toml b/examples/android/Cargo.toml index 1c412bc..cafa8c2 100644 --- a/examples/android/Cargo.toml +++ b/examples/android/Cargo.toml @@ -13,7 +13,7 @@ crate-type = ["rlib", "cdylib"] [dependencies] bevy_oxr.path = "../.." -bevy = "0.12" +bevy = "0.13" openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = ["mint"] } # [profile.release] diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index a52a93c..9b1c664 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -17,7 +17,7 @@ use bevy_oxr::DefaultXrPlugins; #[bevy_main] fn main() { let mut xr_extensions = XrExtensions::default(); - xr_extensions.enable_fb_passthrough(); + // xr_extensions.enable_fb_passthrough(); xr_extensions.enable_hand_tracking(); App::new() .add_plugins(DefaultXrPlugins { @@ -31,7 +31,7 @@ fn main() { .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) .add_plugins(HandInputDebugRenderer) - .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup) + // .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup) .add_systems(Startup, setup) .add_systems( Update, @@ -57,21 +57,21 @@ fn setup( ) { // plane commands.spawn(PbrBundle { - mesh: meshes.add(shape::Plane::from_size(5.0).into()), - material: materials.add(Color::rgb(0.3, 0.5, 0.3).into()), + mesh: meshes.add(Plane3d::new(Vec3::Y)), + material: materials.add(StandardMaterial::from(Color::rgb(0.3, 0.5, 0.3))), ..default() }); // cube commands.spawn(PbrBundle { - mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })), - material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()), + mesh: meshes.add(Cuboid::from_size(Vec3::splat(0.1)).mesh()), + material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))), transform: Transform::from_xyz(0.0, 0.5, 0.0), ..default() }); // cube commands.spawn(PbrBundle { - mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })), - material: materials.add(Color::rgb(0.8, 0.0, 0.0).into()), + mesh: meshes.add(Mesh::from(Cuboid::from_size(Vec3::splat(0.1)))), + material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.0, 0.0))), transform: Transform::from_xyz(0.0, 0.5, 1.0), ..default() }); @@ -106,7 +106,7 @@ fn spawn_controllers_example(mut commands: Commands) { // TODO: make this a vr button fn toggle_passthrough( - keys: Res>, + keys: Res>, passthrough_state: Res, mut resume: EventWriter, mut pause: EventWriter, @@ -114,8 +114,12 @@ fn toggle_passthrough( if keys.just_pressed(KeyCode::Space) { match *passthrough_state { XrPassthroughState::Unsupported => {} - XrPassthroughState::Running => pause.send_default(), - XrPassthroughState::Paused => resume.send_default(), + XrPassthroughState::Running => { + pause.send_default(); + } + XrPassthroughState::Paused => { + resume.send_default(); + } } } } diff --git a/examples/demo/Cargo.toml b/examples/demo/Cargo.toml index 14e39de..00df137 100644 --- a/examples/demo/Cargo.toml +++ b/examples/demo/Cargo.toml @@ -9,9 +9,10 @@ crate-type = ["rlib", "cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bevy = "0.12" +bevy = "0.13" bevy_oxr.path = "../../" -bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } +# bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" } +bevy_rapier3d = "0.25" color-eyre = "0.6.2" diff --git a/examples/demo/src/lib.rs b/examples/demo/src/lib.rs index 8f369f9..2874dc9 100644 --- a/examples/demo/src/lib.rs +++ b/examples/demo/src/lib.rs @@ -1,19 +1,13 @@ use std::{f32::consts::PI, ops::Mul, time::Duration}; use bevy::{ - diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}, - ecs::schedule::ScheduleLabel, - input::{keyboard::KeyCode, Input}, - log::info, - prelude::{ + diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}, ecs::schedule::ScheduleLabel, input::{keyboard::KeyCode, ButtonInput}, log::info, math::primitives::{Capsule3d, Cuboid}, prelude::{ bevy_main, default, shape, App, Assets, Color, Commands, Component, Entity, Event, EventReader, EventWriter, FixedUpdate, Gizmos, GlobalTransform, IntoSystemConfigs, IntoSystemSetConfigs, Mesh, PbrBundle, PostUpdate, Quat, Query, Res, ResMut, Resource, Schedule, SpatialBundle, StandardMaterial, Startup, Transform, Update, Vec3, Vec3Swizzles, With, Without, World, - }, - time::{Fixed, Time, Timer, TimerMode}, - transform::TransformSystem, + }, render::mesh::Meshable, time::{Fixed, Time, Timer, TimerMode}, transform::TransformSystem }; use bevy_oxr::{ graphics::{extensions::XrExtensions, XrAppInfo, XrPreferdBlendMode}, @@ -226,12 +220,8 @@ fn spawn_capsule( ) { commands.spawn(( PbrBundle { - mesh: meshes.add(Mesh::from(shape::Capsule { - radius: 0.033, - depth: 0.115, - ..default() - })), - material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()), + mesh: meshes.add(Capsule3d::new(0.033, 0.115).mesh()), + material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))), transform: Transform::from_xyz(0.0, 2.0, 0.0), ..default() }, @@ -376,7 +366,7 @@ fn update_physics_hands( &Hand, &mut Velocity, )>, - hand_query: Query<(&Transform, &HandBone, &Hand, Without)>, + hand_query: Query<(&Transform, &HandBone, &Hand), Without>, time: Res