From 71a08798ef9f6db71cac672b570ede49c245bf79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Hogsl=C3=A4tt?= <74669087+RasmusHogslatt@users.noreply.github.com> Date: Thu, 1 Feb 2024 05:05:24 +0100 Subject: [PATCH] Working passthrough for Meta Quest 3 (#66) * Window is None * Builds but check manifest * debug prints * Started, not passing last "cvt" * Passthrough works, bevy not visible * Passthrough working * Passthrough working * Working passthrough --- examples/android/manifest.yaml | 12 ++- examples/android/src/lib.rs | 29 ++++-- src/graphics/extensions.rs | 16 +-- src/graphics/mod.rs | 5 +- src/graphics/vulkan.rs | 20 +++- src/lib.rs | 111 +++++++++++++------- src/passthrough.rs | 179 +++++++++++++++++++++++++++++++++ src/resources.rs | 155 ++++++++++++++-------------- src/xr_init.rs | 8 +- 9 files changed, 398 insertions(+), 137 deletions(-) create mode 100644 src/passthrough.rs diff --git a/examples/android/manifest.yaml b/examples/android/manifest.yaml index 9a6118c..7622de3 100644 --- a/examples/android/manifest.yaml +++ b/examples/android/manifest.yaml @@ -3,6 +3,15 @@ android: - "runtime_libs" manifest: package: "org.bevyengine.example_openxr_android" + uses_feature: + - name: "android.hardware.vr.headtracking" + required: true + - name: "oculus.software.handtracking" + required: true + - name: "com.oculus.feature.PASSTHROUGH" + required: true + - name: "com.oculus.experimental.enabled" + required: true application: label: "Bevy Openxr Android" theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen" @@ -14,7 +23,7 @@ android: - name: "com.oculus.supportedDevices" value: "quest|quest2|quest3|questpro" activities: - - config_changes: "density|keyboard|keyboardHidden|navigation|orientation|screenLayout|screenSize|uiMode" + - config_changes: "density|keyboard|keyboardHidden|navigation|orientation|screenLayout|screenSize|uiMode|screenLayout" launch_mode: "singleTask" orientation: "landscape" intent_filters: @@ -23,5 +32,6 @@ android: categories: - "com.oculus.intent.category.VR" - "android.intent.category.LAUNCHER" + - "org.khronos.openxr.intent.category.IMMERSIVE_HMD" sdk: target_sdk_version: 32 diff --git a/examples/android/src/lib.rs b/examples/android/src/lib.rs index 4fabb2b..efb835a 100644 --- a/examples/android/src/lib.rs +++ b/examples/android/src/lib.rs @@ -1,7 +1,11 @@ use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin}; use bevy::prelude::*; use bevy::transform::components::Transform; +use bevy_oxr::graphics::extensions::XrExtensions; use bevy_oxr::graphics::XrAppInfo; +use bevy_oxr::graphics::XrPreferdBlendMode::AlphaBlend; +use bevy_oxr::passthrough::{passthrough_layer_pause, passthrough_layer_resume}; +use bevy_oxr::xr_init::XrRenderData; use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer; use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig}; use bevy_oxr::xr_input::trackers::{ @@ -11,18 +15,21 @@ use bevy_oxr::DefaultXrPlugins; #[bevy_main] fn main() { + let mut xr_extensions = XrExtensions::default(); + xr_extensions.enable_fb_passthrough(); App::new() .add_plugins(DefaultXrPlugins { + reqeusted_extensions: xr_extensions, app_info: XrAppInfo { name: "Bevy OXR Android Example".into(), }, - ..default() + prefered_blend_mode: bevy_oxr::graphics::XrPreferdBlendMode::Opaque, }) .add_plugins(OpenXrDebugRenderer) .add_plugins(LogDiagnosticsPlugin::default()) .add_plugins(FrameTimeDiagnosticsPlugin) .add_systems(Startup, setup) - .add_systems(Update, proto_locomotion) + .add_systems(Update, (proto_locomotion, toggle_passthrough)) .add_systems(Startup, spawn_controllers_example) .insert_resource(PrototypeLocomotionConfig::default()) .run(); @@ -64,11 +71,6 @@ fn setup( transform: Transform::from_xyz(4.0, 8.0, 4.0), ..default() }); - // camera - // commands.spawn((Camera3dBundle { - // transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y), - // ..default() - // },)); } fn spawn_controllers_example(mut commands: Commands) { @@ -87,3 +89,16 @@ fn spawn_controllers_example(mut commands: Commands) { SpatialBundle::default(), )); } + +// Does this work? Not getting logs +fn toggle_passthrough(keys: Res>, mut xr_data: ResMut) { + if keys.just_pressed(KeyCode::Space) { + if xr_data.xr_passthrough_active { + passthrough_layer_pause(xr_data); + bevy::log::info!("Passthrough paused"); + } else { + passthrough_layer_resume(xr_data); + bevy::log::info!("Passthrough resumed"); + } + } +} diff --git a/src/graphics/extensions.rs b/src/graphics/extensions.rs index 2276b79..81d8ca7 100644 --- a/src/graphics/extensions.rs +++ b/src/graphics/extensions.rs @@ -10,14 +10,14 @@ impl XrExtensions { pub fn raw(&self) -> &ExtensionSet { &self.0 } - // pub fn enable_fb_passthrough(&mut self) -> &mut Self { - // self.0.fb_passthrough = true; - // self - // } - // pub fn disable_fb_passthrough(&mut self) -> &mut Self { - // self.0.fb_passthrough = false; - // self - // } + pub fn enable_fb_passthrough(&mut self) -> &mut Self { + self.0.fb_passthrough = true; + self + } + pub fn disable_fb_passthrough(&mut self) -> &mut Self { + self.0.fb_passthrough = false; + self + } pub fn enable_hand_tracking(&mut self) -> &mut Self { self.0.ext_hand_tracking = true; self diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 0aa3dfc..6445986 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -6,9 +6,10 @@ use bevy::window::RawHandleWrapper; use wgpu::Instance; use crate::input::XrInput; +use crate::passthrough::{Passthrough, PassthroughLayer}; use crate::resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrPassthrough, + XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }; use openxr as xr; diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index ba9814e..2678079 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -14,9 +14,12 @@ use xr::EnvironmentBlendMode; use crate::graphics::extensions::XrExtensions; use crate::input::XrInput; + +use crate::passthrough::{Passthrough, PassthroughLayer}; use crate::resources::{ Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, - XrInstance, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrInstance, XrPassthrough, XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, + XrSwapchain, XrViews, }; use crate::VIEW_TYPE; @@ -54,7 +57,7 @@ pub fn initialize_xr_graphics( let available_extensions: XrExtensions = xr_entry.enumerate_extensions()?.into(); assert!(available_extensions.raw().khr_vulkan_enable2); - info!("available xr exts: {:#?}", available_extensions); + //info!("available xr exts: {:#?}", available_extensions); let mut enabled_extensions: xr::ExtensionSet = (available_extensions & reqeusted_extensions).into(); @@ -65,7 +68,7 @@ pub fn initialize_xr_graphics( } let available_layers = xr_entry.enumerate_layers()?; - info!("available xr layers: {:#?}", available_layers); + //info!("available xr layers: {:#?}", available_layers); let xr_instance = xr_entry.create_instance( &xr::ApplicationInfo { @@ -95,20 +98,25 @@ pub fn initialize_xr_graphics( let blend_modes = xr_instance.enumerate_environment_blend_modes(xr_system_id, VIEW_TYPE)?; let blend_mode: EnvironmentBlendMode = match prefered_blend_mode { XrPreferdBlendMode::Opaque if blend_modes.contains(&EnvironmentBlendMode::OPAQUE) => { + bevy::log::info!("Using Opaque"); EnvironmentBlendMode::OPAQUE } XrPreferdBlendMode::Additive if blend_modes.contains(&EnvironmentBlendMode::ADDITIVE) => { + bevy::log::info!("Using Additive"); EnvironmentBlendMode::ADDITIVE } XrPreferdBlendMode::AlphaBlend if blend_modes.contains(&EnvironmentBlendMode::ALPHA_BLEND) => { + bevy::log::info!("Using AlphaBlend"); EnvironmentBlendMode::ALPHA_BLEND } - _ => EnvironmentBlendMode::OPAQUE, + _ => { + bevy::log::info!("Using Opaque"); + EnvironmentBlendMode::OPAQUE + } }; - #[cfg(not(target_os = "android"))] let vk_target_version = vk::make_api_version(0, 1, 2, 0); #[cfg(not(target_os = "android"))] @@ -324,6 +332,8 @@ pub fn initialize_xr_graphics( .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0]) .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb); + // TODO: Log swapchain format + let resolution = uvec2( views[0].recommended_image_rect_width, views[0].recommended_image_rect_height, diff --git a/src/lib.rs b/src/lib.rs index fed5e5b..8a96292 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ pub mod graphics; pub mod input; -// pub mod passthrough; +pub mod passthrough; pub mod resource_macros; pub mod resources; pub mod xr_init; @@ -24,7 +24,7 @@ use graphics::extensions::XrExtensions; use graphics::{XrAppInfo, XrPreferdBlendMode}; use input::XrInput; use openxr as xr; -// use passthrough::{start_passthrough, supports_passthrough, XrPassthroughLayer}; +use passthrough::{start_passthrough, supports_passthrough, Passthrough, PassthroughLayer}; use resources::*; use xr::FormFactor; use xr_init::{xr_only, XrEnableStatus, XrRenderData}; @@ -62,13 +62,16 @@ pub struct FutureXrResources( XrInput, XrViews, XrFrameState, + bool, + XrPassthrough, + XrPassthroughLayer, )>, >, >, ); -// fn mr_test(mut commands: Commands, passthrough_layer: Option>) { -// commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); -// } +fn mr_test(mut commands: Commands, passthrough_layer: Option>) { + commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); +} impl Plugin for OpenXrPlugin { fn build(&self, app: &mut App) { @@ -115,20 +118,60 @@ impl Plugin for OpenXrPlugin { app.insert_resource(input.clone()); app.insert_resource(views.clone()); app.insert_resource(frame_state.clone()); - let xr_data = XrRenderData { - xr_instance, - xr_session: session, - xr_blend_mode: blend_mode, - xr_resolution: resolution, - xr_format: format, - xr_session_running: session_running, - xr_frame_waiter: frame_waiter, - xr_swapchain: swapchain, - xr_input: input, - xr_views: views, - xr_frame_state: frame_state, - }; - app.insert_resource(xr_data); + + // Check if the fb_passthrough extension is available + let fb_passthrough_available = xr_instance.exts().fb_passthrough.is_some(); + bevy::log::info!( + "From OpenXrPlugin: fb_passthrough_available: {}", + fb_passthrough_available + ); + // Get the system for the head-mounted display + let hmd_system = xr_instance + .system(FormFactor::HEAD_MOUNTED_DISPLAY) + .unwrap(); + bevy::log::info!("From OpenXrPlugin: hmd_system: {:?}", hmd_system); + + // Check if the system supports passthrough + let passthrough_supported = + supports_passthrough(&xr_instance, hmd_system).is_ok_and(|v| v); + bevy::log::info!( + "From OpenXrPlugin: passthrough_supported: {}", + passthrough_supported + ); + + // The passthrough variable will be true only if both fb_passthrough is available and the system supports passthrough + let passthrough = fb_passthrough_available && passthrough_supported; + bevy::log::info!("From OpenXrPlugin: passthrough: {}", passthrough); + + let mut p: Option = None; + let mut pl: Option = None; + if passthrough { + if let Ok((p, pl)) = start_passthrough(&xr_instance, &session) { + let xr_data = XrRenderData { + xr_instance, + xr_session: session, + xr_blend_mode: blend_mode, + xr_resolution: resolution, + xr_format: format, + xr_session_running: session_running, + xr_frame_waiter: frame_waiter, + xr_swapchain: swapchain, + xr_input: input, + xr_views: views, + xr_frame_state: frame_state, + xr_passthrough_active: true, + xr_passthrough: XrPassthrough::new(Mutex::new(p)), + xr_passthrough_layer: XrPassthroughLayer::new(Mutex::new(pl)), + }; + bevy::log::info!("Passthrough is supported!"); + app.insert_resource(xr_data); + app.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0))); + } + + if !app.world.contains_resource::() { + info!("ClearColor!"); + } + } app.insert_resource(ActionSets(vec![])); app.add_plugins(RenderPlugin { render_creation: RenderCreation::Manual( @@ -179,23 +222,6 @@ impl Plugin for OpenXrPlugin { } else { app.insert_resource(DisableHandTracking::Both); } - // let passthrough = data.xr_instance.exts().fb_passthrough.is_some() - // && supports_passthrough( - // &data.xr_instance, - // data.xr_instance - // .system(FormFactor::HEAD_MOUNTED_DISPLAY) - // .unwrap(), - // ) - // .is_ok_and(|v| v); - // if passthrough { - // info!("Passthrough!"); - // let (pl, p) = start_passthrough(&data); - // app.insert_resource(pl); - // app.insert_resource(p); - // // if !app.world.contains_resource::() { - // // info!("ClearColor!"); - // // } - // } let (left, right) = data.xr_swapchain.get_render_views(); let left = ManualTextureView { @@ -226,6 +252,8 @@ impl Plugin for OpenXrPlugin { render_app.insert_resource(data.xr_input.clone()); render_app.insert_resource(data.xr_views.clone()); render_app.insert_resource(data.xr_frame_state.clone()); + render_app.insert_resource(data.xr_passthrough.clone()); + render_app.insert_resource(data.xr_passthrough_layer.clone()); render_app.insert_resource(XrEnableStatus::Enabled); render_app.add_systems( Render, @@ -272,7 +300,7 @@ impl PluginGroup for DefaultXrPlugins { ..default() }), #[cfg(target_os = "android")] - primary_window: None, + primary_window: None, // ? #[cfg(target_os = "android")] exit_condition: bevy::window::ExitCondition::DontExit, #[cfg(target_os = "android")] @@ -393,7 +421,7 @@ pub fn end_frame( swapchain: Res, resolution: Res, environment_blend_mode: Res, - // passthrough_layer: Option>, + passthrough_layer: Option>, ) { #[cfg(target_os = "android")] { @@ -408,13 +436,18 @@ pub fn end_frame( } { let _span = info_span!("xr_end_frame").entered(); + // bevy::log::info!( + // "passthrough_layer.is_some(): {:?}", + // passthrough_layer.is_some() + // ); + let result = swapchain.end( xr_frame_state.lock().unwrap().predicted_display_time, &views.lock().unwrap(), &input.stage, **resolution, **environment_blend_mode, - // passthrough_layer.map(|p| p.into_inner()), + passthrough_layer.map(|p| PassthroughLayer(*p.lock().unwrap())), ); match result { Ok(_) => {} diff --git a/src/passthrough.rs b/src/passthrough.rs new file mode 100644 index 0000000..26650e1 --- /dev/null +++ b/src/passthrough.rs @@ -0,0 +1,179 @@ +use bevy::prelude::*; +use std::{marker::PhantomData, mem, ptr}; + +use crate::xr_init::XrRenderData; +use openxr as xr; +use xr::{ + sys::{ + PassthroughCreateInfoFB, PassthroughFB, PassthroughLayerFB, Space, + SystemPassthroughProperties2FB, + }, + CompositionLayerBase, CompositionLayerFlags, Graphics, PassthroughCapabilityFlagsFB, +}; + +use crate::resources::XrInstance; +use crate::resources::XrSession; +pub struct PassthroughLayer(pub xr::sys::PassthroughLayerFB); +pub struct Passthrough(pub xr::sys::PassthroughFB); +fn cvt(x: xr::sys::Result) -> xr::Result { + if x.into_raw() >= 0 { + Ok(x) + } else { + Err(x) + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub(crate) struct CompositionLayerPassthrough<'a, G: xr::Graphics> { + inner: xr::sys::CompositionLayerPassthroughFB, + _marker: PhantomData<&'a G>, +} +impl<'a, G: Graphics> std::ops::Deref for CompositionLayerPassthrough<'a, G> { + type Target = CompositionLayerBase<'a, G>; + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { mem::transmute(&self.inner) } + } +} + +impl<'a, G: xr::Graphics> CompositionLayerPassthrough<'a, G> { + pub(crate) fn from_xr_passthrough_layer(layer: &PassthroughLayer) -> Self { + Self { + inner: xr::sys::CompositionLayerPassthroughFB { + ty: xr::sys::CompositionLayerPassthroughFB::TYPE, + next: ptr::null(), + flags: CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA, + space: Space::NULL, + layer_handle: layer.0, + }, + _marker: PhantomData, + } + } +} +#[inline] +pub fn supports_passthrough(instance: &XrInstance, system: xr::SystemId) -> xr::Result { + unsafe { + let mut hand = xr::sys::SystemPassthroughProperties2FB { + ty: SystemPassthroughProperties2FB::TYPE, + next: ptr::null(), + capabilities: PassthroughCapabilityFlagsFB::PASSTHROUGH_CAPABILITY, + }; + let mut p = xr::sys::SystemProperties::out(&mut hand as *mut _ as _); + cvt((instance.fp().get_system_properties)( + instance.as_raw(), + system, + p.as_mut_ptr(), + ))?; + bevy::log::info!( + "From supports_passthrough: Passthrough capabilities: {:?}", + hand.capabilities + ); + Ok( + (hand.capabilities & PassthroughCapabilityFlagsFB::PASSTHROUGH_CAPABILITY) + == PassthroughCapabilityFlagsFB::PASSTHROUGH_CAPABILITY, + ) + } +} + +#[inline] +pub fn start_passthrough( + instance: &XrInstance, + xr_session: &XrSession, +) -> xr::Result<(xr::sys::PassthroughFB, xr::sys::PassthroughLayerFB)> { + unsafe { + // Create feature + let mut passthrough_feature = xr::sys::PassthroughFB::NULL; + let mut passthrough_create_info = xr::sys::PassthroughCreateInfoFB { + ty: xr::sys::StructureType::PASSTHROUGH_CREATE_INFO_FB, // XR_TYPE_PASSTHROUGH_CREATE_INFO_FB + next: ptr::null(), + flags: xr::sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION, + }; + // bevy::log::info!("xr_session.as_raw(): {:?}", xr_session.as_raw()); + // bevy::log::info!("&passthrough_create_info: {:?}", &passthrough_create_info); + // bevy::log::info!("&mut passthrough_feature: {:?}", &mut passthrough_feature); + // bevy::log::info!( + // "instance.exts().fb_passthrough.unwrap(): {:?}", + // instance.exts().fb_passthrough.is_some() + // ); + cvt( + (instance.exts().fb_passthrough.unwrap().create_passthrough)( + xr_session.as_raw(), + &passthrough_create_info as *const _, + &mut passthrough_feature as *mut _, + ), + )?; + // bevy::log::info!("Created passthrough feature"); + // Create layer + let mut passthrough_layer = xr::sys::PassthroughLayerFB::NULL; + let mut layer_create_info: xr::sys::PassthroughLayerCreateInfoFB = + xr::sys::PassthroughLayerCreateInfoFB { + ty: xr::sys::StructureType::PASSTHROUGH_LAYER_CREATE_INFO_FB, // XR_TYPE_PASSTHROUGH_LAYER_CREATE_INFO_FB + next: ptr::null(), + passthrough: passthrough_feature, // XR_PASSTHROUGH_HANDLE + flags: xr::sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION, // XR_PASSTHROUGH_IS_RUNNING_AT_CREATION_BIT_FB + purpose: xr::sys::PassthroughLayerPurposeFB::RECONSTRUCTION, // XR_PASSTHROUGH_LAYER_PURPOSE_RECONSTRUCTION_FB + }; + cvt((instance + .exts() + .fb_passthrough + .unwrap() + .create_passthrough_layer)( + xr_session.as_raw(), + &layer_create_info as *const _, + &mut passthrough_layer as *mut _, + ))?; + // bevy::log::info!("Created passthrough layer"); + // // Start layer + + // bevy::log::info!("passthrough_feature: {:?}", passthrough_feature); + // // cvt((instance.exts().fb_passthrough.unwrap().passthrough_start)( + // // passthrough_feature, + // // ))?; + // bevy::log::info!("Started passthrough layer"); + // bevy::log::info!("Passed everything in start"); + Ok((passthrough_feature, passthrough_layer)) + } +} + +#[inline] +pub fn passthrough_layer_resume(mut xr_data_resource: ResMut) -> xr::Result<()> { + unsafe { + let passthrough_layer = &xr_data_resource.xr_passthrough_layer; + { + let passthrough_layer_locked = passthrough_layer.lock().unwrap(); + cvt((xr_data_resource + .xr_instance + .exts() + .fb_passthrough + .unwrap() + .passthrough_layer_resume)( + *passthrough_layer_locked + ))?; + } + xr_data_resource.xr_passthrough_active = true; + bevy::log::info!("Resumed passthrough layer"); + Ok(()) + } +} + +#[inline] +pub fn passthrough_layer_pause(mut xr_data_resource: ResMut) -> xr::Result<()> { + unsafe { + let passthrough_layer = &xr_data_resource.xr_passthrough_layer; + { + let passthrough_layer_locked = passthrough_layer.lock().unwrap(); + cvt((xr_data_resource + .xr_instance + .exts() + .fb_passthrough + .unwrap() + .passthrough_layer_pause)( + *passthrough_layer_locked + ))?; + } + xr_data_resource.xr_passthrough_active = false; + bevy::log::info!("Paused passthrough layer"); + Ok(()) + } +} diff --git a/src/resources.rs b/src/resources.rs index a798955..af8fb66 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -1,9 +1,12 @@ use std::sync::atomic::AtomicBool; use std::sync::Mutex; -// use crate::passthrough::XrPassthroughLayer; +use crate::passthrough::{Passthrough, PassthroughLayer}; use crate::resource_macros::*; +use crate::xr::sys::CompositionLayerPassthroughFB; +use crate::xr::{CompositionLayerBase, CompositionLayerFlags}; use bevy::prelude::*; +use core::ptr; use openxr as xr; xr_resource_wrapper!(XrInstance, xr::Instance); @@ -16,6 +19,8 @@ xr_arc_resource_wrapper!(XrFrameWaiter, Mutex); xr_arc_resource_wrapper!(XrSwapchain, Swapchain); xr_arc_resource_wrapper!(XrFrameState, Mutex); xr_arc_resource_wrapper!(XrViews, Mutex>); +xr_arc_resource_wrapper!(XrPassthrough, Mutex); +xr_arc_resource_wrapper!(XrPassthroughLayer, Mutex); pub enum Swapchain { Vulkan(SwapchainInner), @@ -59,7 +64,7 @@ impl Swapchain { stage: &xr::Space, resolution: UVec2, environment_blend_mode: xr::EnvironmentBlendMode, - // passthrough_layer: Option<&XrPassthroughLayer>, + passthrough_layer: Option, ) -> xr::Result<()> { match self { Swapchain::Vulkan(swapchain) => swapchain.end( @@ -68,7 +73,7 @@ impl Swapchain { stage, resolution, environment_blend_mode, - // passthrough_layer, + passthrough_layer, ), } } @@ -128,7 +133,7 @@ impl SwapchainInner { stage: &xr::Space, resolution: UVec2, environment_blend_mode: xr::EnvironmentBlendMode, - // passthrough_layer: Option<&XrPassthroughLayer>, + passthrough_layer: Option, ) -> xr::Result<()> { let rect = xr::Rect2Di { offset: xr::Offset2Di { x: 0, y: 0 }, @@ -142,75 +147,79 @@ impl SwapchainInner { warn!("views are len of 0"); return Ok(()); } - // match passthrough_layer { - // Some(pass) => { - // // info!("Rendering with pass through"); - // let passthrough_layer = xr::sys::CompositionLayerPassthroughFB { - // ty: CompositionLayerPassthroughFB::TYPE, - // next: ptr::null(), - // flags: CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA, - // space: xr::sys::Space::NULL, - // layer_handle: pass.0, - // }; - // self.stream.lock().unwrap().end( - // predicted_display_time, - // environment_blend_mode, - // &[ - // &xr::CompositionLayerProjection::new() - // .layer_flags(CompositionLayerFlags::UNPREMULTIPLIED_ALPHA) - // .space(stage) - // .views(&[ - // xr::CompositionLayerProjectionView::new() - // .pose(views[0].pose) - // .fov(views[0].fov) - // .sub_image( - // xr::SwapchainSubImage::new() - // .swapchain(&swapchain) - // .image_array_index(0) - // .image_rect(rect), - // ), - // xr::CompositionLayerProjectionView::new() - // .pose(views[1].pose) - // .fov(views[1].fov) - // .sub_image( - // xr::SwapchainSubImage::new() - // .swapchain(&swapchain) - // .image_array_index(1) - // .image_rect(rect), - // ), - // ]), - // unsafe { - // &*(&passthrough_layer as *const _ as *const CompositionLayerBase) - // }, - // ], - // ) - // } + match passthrough_layer { + Some(pass) => { + //bevy::log::info!("Rendering with pass through"); - // None => - self.stream.lock().unwrap().end( - predicted_display_time, - environment_blend_mode, - &[&xr::CompositionLayerProjection::new().space(stage).views(&[ - xr::CompositionLayerProjectionView::new() - .pose(views[0].pose) - .fov(views[0].fov) - .sub_image( - xr::SwapchainSubImage::new() - .swapchain(&swapchain) - .image_array_index(0) - .image_rect(rect), - ), - xr::CompositionLayerProjectionView::new() - .pose(views[1].pose) - .fov(views[1].fov) - .sub_image( - xr::SwapchainSubImage::new() - .swapchain(&swapchain) - .image_array_index(1) - .image_rect(rect), - ), - ])], - ) - // } + let passthrough_layer = xr::sys::CompositionLayerPassthroughFB { + ty: CompositionLayerPassthroughFB::TYPE, + next: ptr::null(), + flags: CompositionLayerFlags::UNPREMULTIPLIED_ALPHA, + space: xr::sys::Space::NULL, + layer_handle: pass.0, + }; + + self.stream.lock().unwrap().end( + predicted_display_time, + environment_blend_mode, + &[ + unsafe { + &*(&passthrough_layer as *const _ as *const CompositionLayerBase) + }, + &xr::CompositionLayerProjection::new() + .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA) + .space(stage) + .views(&[ + xr::CompositionLayerProjectionView::new() + .pose(views[0].pose) + .fov(views[0].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(0) + .image_rect(rect), + ), + xr::CompositionLayerProjectionView::new() + .pose(views[1].pose) + .fov(views[1].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(1) + .image_rect(rect), + ), + ]), + ], + ) + } + + None => { + bevy::log::info!("Rendering without pass through"); + self.stream.lock().unwrap().end( + predicted_display_time, + environment_blend_mode, + &[&xr::CompositionLayerProjection::new().space(stage).views(&[ + xr::CompositionLayerProjectionView::new() + .pose(views[0].pose) + .fov(views[0].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(0) + .image_rect(rect), + ), + xr::CompositionLayerProjectionView::new() + .pose(views[1].pose) + .fov(views[1].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(1) + .image_rect(rect), + ), + ])], + ) + } + } } } diff --git a/src/xr_init.rs b/src/xr_init.rs index 499e7ce..88315c0 100644 --- a/src/xr_init.rs +++ b/src/xr_init.rs @@ -17,9 +17,10 @@ use wgpu::Instance; use crate::{ input::XrInput, + passthrough::{Passthrough, PassthroughLayer}, resources::{ - XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution, - XrSession, XrSessionRunning, XrSwapchain, XrViews, + XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrPassthrough, + XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews, }, }; @@ -45,6 +46,9 @@ pub struct XrRenderData { pub xr_input: XrInput, pub xr_views: XrViews, pub xr_frame_state: XrFrameState, + pub xr_passthrough_active: bool, + pub xr_passthrough: XrPassthrough, + pub xr_passthrough_layer: XrPassthroughLayer, } #[derive(Event, Clone, Copy, Debug)]