From 13890ddb53f7c733934f59a834eb3fcda8e350be Mon Sep 17 00:00:00 2001 From: awtterpip Date: Fri, 3 May 2024 19:31:23 -0500 Subject: [PATCH] basic passthrough stuff --- .../bevy_openxr/examples/android/Cargo.toml | 5 + .../bevy_openxr/examples/android/src/lib.rs | 1 + .../src/openxr/features/passthrough.rs | 60 ++++++- .../bevy_openxr/src/openxr/layer_builder.rs | 71 +++++++- crates/bevy_openxr/src/openxr/mod.rs | 9 +- crates/bevy_openxr/src/openxr/render.rs | 155 +++++++++++------- crates/bevy_openxr/src/openxr/resources.rs | 61 +++---- 7 files changed, 261 insertions(+), 101 deletions(-) diff --git a/crates/bevy_openxr/examples/android/Cargo.toml b/crates/bevy_openxr/examples/android/Cargo.toml index 10e5235..fdd2821 100644 --- a/crates/bevy_openxr/examples/android/Cargo.toml +++ b/crates/bevy_openxr/examples/android/Cargo.toml @@ -41,6 +41,11 @@ launch_mode = "singleTask" orientation = "landscape" resizeable_activity = false +[[package.metadata.android.uses_feature]] +name = "com.oculus.feature.PASSTHROUGH" +required = true +version = 1 + [[package.metadata.android.application.activity.intent_filter]] actions = ["android.intent.action.MAIN"] categories = [ diff --git a/crates/bevy_openxr/examples/android/src/lib.rs b/crates/bevy_openxr/examples/android/src/lib.rs index c9a7190..6f60261 100644 --- a/crates/bevy_openxr/examples/android/src/lib.rs +++ b/crates/bevy_openxr/examples/android/src/lib.rs @@ -8,6 +8,7 @@ fn main() { App::new() .add_plugins(add_xr_plugins(DefaultPlugins)) .add_systems(Startup, setup) + .insert_resource(ClearColor(Color::NONE)) .run(); } diff --git a/crates/bevy_openxr/src/openxr/features/passthrough.rs b/crates/bevy_openxr/src/openxr/features/passthrough.rs index 948c9d1..3262210 100644 --- a/crates/bevy_openxr/src/openxr/features/passthrough.rs +++ b/crates/bevy_openxr/src/openxr/features/passthrough.rs @@ -1,18 +1,74 @@ use bevy::prelude::*; +use bevy::render::Render; +use bevy::render::RenderApp; +use bevy::render::RenderSet; use openxr::sys::SystemPassthroughProperties2FB; use openxr::PassthroughCapabilityFlagsFB; +use crate::layer_builder::PassthroughLayer; use crate::resources::*; use crate::types::*; pub struct OxrPassthroughPlugin; impl Plugin for OxrPassthroughPlugin { - fn build(&self, _app: &mut App) { - todo!() + fn build(&self, app: &mut App) { + let resources = app + .world + .get_resource::() + .and_then(|instance| { + app.world + .get_resource::() + .map(|system_id| (instance, system_id)) + }); + if resources.is_some_and(|(instance, system)| { + supports_passthrough(instance, *system).is_ok_and(|s| s) + }) { + app.sub_app_mut(RenderApp).add_systems( + Render, + insert_passthrough + .in_set(RenderSet::PrepareAssets) + .run_if(resource_added::), + ); + } else { + error!("Passthrough is not supported with this runtime") + } } } +pub fn insert_passthrough(world: &mut World) { + let session = world.resource::(); + + let (passthrough, passthrough_layer) = create_passthrough( + session, + openxr::PassthroughFlagsFB::IS_RUNNING_AT_CREATION, + openxr::PassthroughLayerPurposeFB::RECONSTRUCTION, + ) + .unwrap(); + + world + .resource_mut::() + .insert(0, Box::new(PassthroughLayer)); + world.insert_resource(passthrough); + world.insert_resource(passthrough_layer); +} + +pub fn resume_passthrough( + passthrough: Res, + passthrough_layer: Res, +) { + passthrough.start().unwrap(); + passthrough_layer.resume().unwrap(); +} + +pub fn pause_passthrough( + passthrough: Res, + passthrough_layer: Res, +) { + passthrough_layer.pause().unwrap(); + passthrough.pause().unwrap(); +} + pub fn create_passthrough( session: &OxrSession, flags: openxr::PassthroughFlagsFB, diff --git a/crates/bevy_openxr/src/openxr/layer_builder.rs b/crates/bevy_openxr/src/openxr/layer_builder.rs index 531b5fc..a9a89d0 100644 --- a/crates/bevy_openxr/src/openxr/layer_builder.rs +++ b/crates/bevy_openxr/src/openxr/layer_builder.rs @@ -1,9 +1,70 @@ use std::mem; +use bevy::ecs::world::World; use openxr::{sys, CompositionLayerFlags, Fovf, Posef, Rect2Di, Space}; use crate::graphics::graphics_match; -use crate::resources::{OxrPassthroughLayer, OxrSwapchain}; +use crate::resources::*; + +pub trait LayerProvider { + fn get<'a>(&'a self, world: &'a World) -> Box; +} + +pub struct ProjectionLayer; + +pub struct PassthroughLayer; + +impl LayerProvider for ProjectionLayer { + fn get<'a>(&self, world: &'a World) -> Box + 'a> { + let stage = world.resource::(); + let openxr_views = world.resource::(); + let swapchain = world.resource::(); + let graphics_info = world.resource::(); + let rect = openxr::Rect2Di { + offset: openxr::Offset2Di { x: 0, y: 0 }, + extent: openxr::Extent2Di { + width: graphics_info.resolution.x as _, + height: graphics_info.resolution.y as _, + }, + }; + + Box::new( + CompositionLayerProjection::new() + .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA) + .space(&stage) + .views(&[ + CompositionLayerProjectionView::new() + .pose(openxr_views.0[0].pose) + .fov(openxr_views.0[0].fov) + .sub_image( + SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(0) + .image_rect(rect), + ), + CompositionLayerProjectionView::new() + .pose(openxr_views.0[1].pose) + .fov(openxr_views.0[1].fov) + .sub_image( + SwapchainSubImage::new() + .swapchain(&swapchain) + .image_array_index(1) + .image_rect(rect), + ), + ]), + ) + } +} + +impl LayerProvider for PassthroughLayer { + fn get<'a>(&'a self, world: &'a World) -> Box { + Box::new( + CompositionLayerPassthrough::new() + .layer_handle(world.resource::()) + .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA), + ) + } +} #[derive(Copy, Clone)] pub struct SwapchainSubImage<'a> { @@ -144,12 +205,10 @@ impl<'a> CompositionLayerProjection<'a> { self } #[inline] - pub fn views(mut self, value: &'a [CompositionLayerProjectionView<'a>]) -> Self { - for view in value { - self.views.push(view.inner.clone()); - } + pub fn views(mut self, value: &[CompositionLayerProjectionView<'a>]) -> Self { + self.views = value.iter().map(|view| view.inner).collect(); self.inner.views = self.views.as_slice().as_ptr() as *const _ as _; - self.inner.view_count = value.len() as u32; + self.inner.view_count = self.views.len() as u32; self } } diff --git a/crates/bevy_openxr/src/openxr/mod.rs b/crates/bevy_openxr/src/openxr/mod.rs index 328c290..441e8bd 100644 --- a/crates/bevy_openxr/src/openxr/mod.rs +++ b/crates/bevy_openxr/src/openxr/mod.rs @@ -10,6 +10,8 @@ use bevy_xr::session::XrSessionPlugin; use init::OxrInitPlugin; use render::OxrRenderPlugin; +use self::{exts::OxrExtensions, features::passthrough::OxrPassthroughPlugin}; + pub mod error; mod exts; pub mod features; @@ -28,7 +30,11 @@ pub fn add_xr_plugins(plugins: G) -> PluginGroupBuilder { .add_before::(XrSessionPlugin) .add_before::(OxrInitPlugin { app_info: default(), - exts: default(), + exts: { + let mut exts = OxrExtensions::default(); + exts.enable_fb_passthrough(); + exts + }, blend_modes: default(), backends: default(), formats: Some(vec![wgpu::TextureFormat::Rgba8UnormSrgb]), @@ -36,6 +42,7 @@ pub fn add_xr_plugins(plugins: G) -> PluginGroupBuilder { synchronous_pipeline_compilation: default(), }) .add(OxrRenderPlugin) + .add(OxrPassthroughPlugin) .add(XrCameraPlugin) // .add(XrActionPlugin) .set(WindowPlugin { diff --git a/crates/bevy_openxr/src/openxr/render.rs b/crates/bevy_openxr/src/openxr/render.rs index 6c060b8..d9172c7 100644 --- a/crates/bevy_openxr/src/openxr/render.rs +++ b/crates/bevy_openxr/src/openxr/render.rs @@ -10,11 +10,13 @@ use bevy::{ transform::TransformSystem, }; use bevy_xr::camera::{XrCamera, XrCameraBundle, XrProjection}; -use openxr::{CompositionLayerFlags, ViewStateFlags}; +use openxr::ViewStateFlags; -use crate::init::{session_started, OxrPreUpdateSet}; -use crate::layer_builder::*; use crate::resources::*; +use crate::{ + init::{session_started, OxrPreUpdateSet}, + layer_builder::ProjectionLayer, +}; pub struct OxrRenderPlugin; @@ -39,24 +41,28 @@ impl Plugin for OxrRenderPlugin { .run_if(session_started) .before(TransformSystem::TransformPropagate), ); - app.sub_app_mut(RenderApp).add_systems( - Render, - ( + app.sub_app_mut(RenderApp) + .add_systems( + Render, ( - insert_texture_views, - locate_views, - update_views_render_world, + ( + insert_texture_views, + locate_views, + update_views_render_world, + ) + .chain() + .in_set(RenderSet::PrepareAssets), + begin_frame + .before(RenderSet::Queue) + .before(insert_texture_views), + wait_image.in_set(RenderSet::Render).before(render_system), + (release_image, end_frame) + .chain() + .in_set(RenderSet::Cleanup), ) - .chain() - .in_set(RenderSet::PrepareAssets), - begin_frame - .before(RenderSet::Queue) - .before(insert_texture_views), - wait_image.in_set(RenderSet::Render).before(render_system), - (end_frame).chain().in_set(RenderSet::Cleanup), + .run_if(session_started), ) - .run_if(session_started), - ); + .insert_resource(OxrRenderLayers(vec![Box::new(ProjectionLayer)])); } } @@ -326,50 +332,73 @@ pub fn begin_frame(mut frame_stream: ResMut) { frame_stream.begin().expect("Failed to begin frame") } -pub fn end_frame( - mut frame_stream: ResMut, - mut swapchain: ResMut, - stage: Res, - display_time: Res, - graphics_info: Res, - openxr_views: Res, -) { - let _span = info_span!("xr_end_frame"); +pub fn release_image(mut swapchain: ResMut) { + let _span = info_span!("xr_release_image"); swapchain.release_image().unwrap(); - let rect = openxr::Rect2Di { - offset: openxr::Offset2Di { x: 0, y: 0 }, - extent: openxr::Extent2Di { - width: graphics_info.resolution.x as _, - height: graphics_info.resolution.y as _, - }, - }; - frame_stream - .end( - **display_time, - graphics_info.blend_mode, - &[&CompositionLayerProjection::new() - .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA) - .space(&stage) - .views(&[ - CompositionLayerProjectionView::new() - .pose(openxr_views.0[0].pose) - .fov(openxr_views.0[0].fov) - .sub_image( - SwapchainSubImage::new() - .swapchain(&swapchain) - .image_array_index(0) - .image_rect(rect), - ), - CompositionLayerProjectionView::new() - .pose(openxr_views.0[1].pose) - .fov(openxr_views.0[1].fov) - .sub_image( - SwapchainSubImage::new() - .swapchain(&swapchain) - .image_array_index(1) - .image_rect(rect), - ), - ])], - ) - .expect("Failed to end frame"); } + +pub fn end_frame(world: &mut World) { + let _span = info_span!("xr_end_frame"); + world.resource_scope::(|world, mut frame_stream| { + let mut layers = vec![]; + for layer in world.resource::().iter() { + layers.push(layer.get(world)); + } + let layers: Vec<_> = layers.iter().map(Box::as_ref).collect(); + frame_stream + .end( + **world.resource::(), + world.resource::().blend_mode, + &layers, + ) + .expect("Failed to end frame"); + }); +} + +// pub fn end_frame( +// mut frame_stream: ResMut, +// mut swapchain: ResMut, +// stage: Res, +// display_time: Res, +// graphics_info: Res, +// openxr_views: Res, +// ) { +// let _span = info_span!("xr_end_frame"); +// swapchain.release_image().unwrap(); +// let rect = openxr::Rect2Di { +// offset: openxr::Offset2Di { x: 0, y: 0 }, +// extent: openxr::Extent2Di { +// width: graphics_info.resolution.x as _, +// height: graphics_info.resolution.y as _, +// }, +// }; +// frame_stream +// .end( +// **display_time, +// graphics_info.blend_mode, +// &[&CompositionLayerProjection::new() +// .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA) +// .space(&stage) +// .views(&[ +// CompositionLayerProjectionView::new() +// .pose(openxr_views.0[0].pose) +// .fov(openxr_views.0[0].fov) +// .sub_image( +// SwapchainSubImage::new() +// .swapchain(&swapchain) +// .image_array_index(0) +// .image_rect(rect), +// ), +// CompositionLayerProjectionView::new() +// .pose(openxr_views.0[1].pose) +// .fov(openxr_views.0[1].fov) +// .sub_image( +// SwapchainSubImage::new() +// .swapchain(&swapchain) +// .image_array_index(1) +// .image_rect(rect), +// ), +// ])], +// ) +// .expect("Failed to end frame"); +// } diff --git a/crates/bevy_openxr/src/openxr/resources.rs b/crates/bevy_openxr/src/openxr/resources.rs index bbf814a..b51d283 100644 --- a/crates/bevy_openxr/src/openxr/resources.rs +++ b/crates/bevy_openxr/src/openxr/resources.rs @@ -7,7 +7,7 @@ use openxr::AnyGraphics; use crate::error::OxrError; use crate::graphics::*; -use crate::layer_builder::CompositionLayer; +use crate::layer_builder::{CompositionLayer, LayerProvider}; use crate::types::*; /// Wrapper around an [`Entry`](openxr::Entry) with some methods overridden to use bevy types. @@ -237,34 +237,6 @@ impl OxrSession { } } -/// Wrapper around [`openxr::Passthrough`]. -/// -/// Used to [`start`](openxr::Passthrough::start) or [`pause`](openxr::Passthrough::pause) passthrough on the physical device. -/// -/// See [`openxr::Passthrough`] for available methods. -#[derive(Resource, Deref, DerefMut)] -pub struct OxrPassthrough( - #[deref] pub openxr::Passthrough, - /// The flags are stored here so that they don't need to be passed in again when creating an [`OxrPassthroughLayer`]. - openxr::PassthroughFlagsFB, -); - -impl OxrPassthrough { - /// This function can create an [`OxrPassthrough`] from raw openxr types if needed. - /// In the majority of cases, you should use [`create_passthrough`](OxrSession::create_passthrough) instead. - pub fn from_inner(passthrough: openxr::Passthrough, flags: openxr::PassthroughFlagsFB) -> Self { - Self(passthrough, flags) - } -} - -/// Wrapper around [`openxr::Passthrough`]. -/// -/// Used to create a [`CompositionLayerPassthrough`](crate::layer_builder::CompositionLayerPassthrough), and to [`pause`](openxr::PassthroughLayer::pause) or [`resume`](openxr::PassthroughLayer::resume) rendering of the passthrough layer. -/// -/// See [`openxr::PassthroughLayer`] for available methods. -#[derive(Resource, Deref, DerefMut)] -pub struct OxrPassthroughLayer(pub openxr::PassthroughLayer); - /// Graphics agnostic wrapper around [openxr::FrameStream] #[derive(Resource)] pub struct OxrFrameStream(pub GraphicsWrap); @@ -416,6 +388,37 @@ pub struct OxrViews(pub Vec); #[derive(Debug, Copy, Clone, Deref, Default, Eq, PartialEq, Ord, PartialOrd, Hash, Resource)] pub struct OxrSystemId(pub openxr::SystemId); +/// Wrapper around [`openxr::Passthrough`]. +/// +/// Used to [`start`](openxr::Passthrough::start) or [`pause`](openxr::Passthrough::pause) passthrough on the physical device. +/// +/// See [`openxr::Passthrough`] for available methods. +#[derive(Resource, Deref, DerefMut)] +pub struct OxrPassthrough( + #[deref] pub openxr::Passthrough, + /// The flags are stored here so that they don't need to be passed in again when creating an [`OxrPassthroughLayer`]. + openxr::PassthroughFlagsFB, +); + +impl OxrPassthrough { + /// This function can create an [`OxrPassthrough`] from raw openxr types if needed. + /// In the majority of cases, you should use [`create_passthrough`](OxrSession::create_passthrough) instead. + pub fn from_inner(passthrough: openxr::Passthrough, flags: openxr::PassthroughFlagsFB) -> Self { + Self(passthrough, flags) + } +} + +/// Wrapper around [`openxr::Passthrough`]. +/// +/// Used to create a [`CompositionLayerPassthrough`](crate::layer_builder::CompositionLayerPassthrough), and to [`pause`](openxr::PassthroughLayer::pause) or [`resume`](openxr::PassthroughLayer::resume) rendering of the passthrough layer. +/// +/// See [`openxr::PassthroughLayer`] for available methods. +#[derive(Resource, Deref, DerefMut)] +pub struct OxrPassthroughLayer(pub openxr::PassthroughLayer); + +#[derive(Resource, Deref, DerefMut, Default)] +pub struct OxrRenderLayers(pub Vec>); + /// Resource storing graphics info for the currently running session. #[derive(Clone, Copy, Resource)] pub struct OxrGraphicsInfo {