session restarting workscargo run --release --example xr! views fixed and late latching for views

examples/xr.rs

@@ -6,7 +6,7 @@ use bevy_oxr::graphics::XrAppInfo;
 use bevy_oxr::input::XrInput;
 use bevy_oxr::resources::{XrFrameState, XrSession};
 
-use bevy_oxr::xr_init::{xr_only, EndXrSession, StartXrSession};
+use bevy_oxr::xr_init::{xr_only, EndXrSession, StartXrSession, XrSetup};
 use bevy_oxr::xr_input::actions::XrActionSets;
 use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer;
 use bevy_oxr::xr_input::interactions::{
@@ -37,35 +37,35 @@ fn main() {
         .add_plugins(LogDiagnosticsPlugin::default())
         .add_plugins(FrameTimeDiagnosticsPlugin)
         .add_systems(Startup, setup)
-        // .add_systems(Update, proto_locomotion.run_if(xr_only()))
+        .add_systems(Update, proto_locomotion.run_if(xr_only()))
-        // .insert_resource(PrototypeLocomotionConfig::default())
+        .insert_resource(PrototypeLocomotionConfig::default())
-        .add_systems(Startup, spawn_controllers_example)
+        .add_systems(XrSetup, spawn_controllers_example)
-        // .add_plugins(HandInputDebugRenderer)
+        .add_plugins(HandInputDebugRenderer)
-        // .add_systems(
+        .add_systems(
-        //     Update,
+            Update,
-        //     draw_interaction_gizmos
+            draw_interaction_gizmos
-        //         .after(update_interactable_states)
+                .after(update_interactable_states)
-        //         .run_if(xr_only()),
+                .run_if(xr_only()),
-        // )
+        )
-        // .add_systems(
+        .add_systems(
-        //     Update,
+            Update,
-        //     draw_socket_gizmos
+            draw_socket_gizmos
-        //         .after(update_interactable_states)
+                .after(update_interactable_states)
-        //         .run_if(xr_only()),
+                .run_if(xr_only()),
-        // )
+        )
-        // .add_systems(
+        .add_systems(
-        //     Update,
+            Update,
-        //     interactions
+            interactions
-        //         .before(update_interactable_states)
+                .before(update_interactable_states)
-        //         .run_if(xr_only()),
+                .run_if(xr_only()),
-        // )
+        )
-        // .add_systems(
+        .add_systems(
-        //     Update,
+            Update,
-        //     socket_interactions.before(update_interactable_states),
+            socket_interactions.before(update_interactable_states),
-        // )
+        )
-        // .add_systems(Update, prototype_interaction_input.run_if(xr_only()))
+        .add_systems(Update, prototype_interaction_input.run_if(xr_only()))
-        // .add_systems(Update, update_interactable_states)
+        .add_systems(Update, update_interactable_states)
-        // .add_systems(Update, update_grabbables.after(update_interactable_states))
+        .add_systems(Update, update_grabbables.after(update_interactable_states))
         .add_systems(Update, start_stop_session)
         .add_event::<InteractionEvent>()
         .run();
@@ -170,13 +170,14 @@ fn spawn_controllers_example(mut commands: Commands) {
     ));
 }
 
+#[allow(clippy::type_complexity)]
 fn prototype_interaction_input(
     oculus_controller: Res<OculusController>,
     frame_state: Res<XrFrameState>,
     xr_input: Res<XrInput>,
     session: Res<XrSession>,
     mut right_interactor_query: Query<
-        (&mut XRInteractorState),
+        &mut XRInteractorState,
         (
             With<XRDirectInteractor>,
             With<OpenXRRightController>,
@@ -184,7 +185,7 @@ fn prototype_interaction_input(
         ),
     >,
     mut left_interactor_query: Query<
-        (&mut XRInteractorState),
+        &mut XRInteractorState,
         (
             With<XRRayInteractor>,
             With<OpenXRLeftController>,

src/lib.rs

@@ -11,6 +11,7 @@ use std::sync::atomic::AtomicBool;
 
 use crate::xr_init::{StartXrSession, XrInitPlugin};
 use crate::xr_input::oculus_touch::ActionSets;
+use crate::xr_input::trackers::verify_quat;
 use bevy::app::{AppExit, PluginGroupBuilder};
 use bevy::core::TaskPoolThreadAssignmentPolicy;
 use bevy::ecs::system::SystemState;
@@ -36,6 +37,7 @@ use xr_input::hands::emulated::HandEmulationPlugin;
 use xr_input::hands::hand_tracking::HandTrackingPlugin;
 use xr_input::hands::HandPlugin;
 use xr_input::xr_camera::XrCameraPlugin;
+use xr_input::XrInputPlugin;
 
 const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
 
@@ -129,15 +131,15 @@ impl Plugin for OpenXrPlugin {
                 .after(xr_poll_events),
         );
         let render_app = app.sub_app_mut(RenderApp);
-        render_app.add_systems(
+        // render_app.add_systems(
-            Render,
+        //     Render,
-            xr_begin_frame
+        //     xr_begin_frame
-                .run_if(xr_only())
+        //         .run_if(xr_only())
-                .run_if(xr_after_wait_only())
+        //         .run_if(xr_after_wait_only())
-                // .run_if(xr_render_only())
+        //         // .run_if(xr_render_only())
-                .after(RenderSet::ExtractCommands)
+        //         .after(RenderSet::ExtractCommands)
-                .before(xr_pre_frame),
+        //         .before(xr_pre_frame),
-        );
+        // );
         render_app.add_systems(
             Render,
             xr_pre_frame
@@ -174,7 +176,8 @@ impl Plugin for OpenXrPlugin {
 }
 
 fn clean_resources_render(mut cmds: &mut World) {
-    let session = cmds.remove_resource::<XrSession>().unwrap();
+    // let session = cmds.remove_resource::<XrSession>().unwrap();
+    cmds.remove_resource::<XrSession>();
     cmds.remove_resource::<XrResolution>();
     cmds.remove_resource::<XrFormat>();
     // cmds.remove_resource::<XrSessionRunning>();
@@ -190,7 +193,7 @@ fn clean_resources_render(mut cmds: &mut World) {
     warn!("Cleanup Resources Render");
 }
 fn clean_resources(mut cmds: &mut World) {
-    let session = cmds.remove_resource::<XrSession>().unwrap();
+    cmds.remove_resource::<XrSession>();
     cmds.remove_resource::<XrResolution>();
     cmds.remove_resource::<XrFormat>();
     // cmds.remove_resource::<XrSessionRunning>();
@@ -257,14 +260,14 @@ impl PluginGroup for DefaultXrPlugins {
                 app_info: self.app_info.clone(),
             })
             .add_after::<OpenXrPlugin, _>(XrInitPlugin)
-            // .add(XrInput)
+            .add(XrInputPlugin)
-            // .add(XrActionsPlugin)
+            .add(XrActionsPlugin)
             .add(XrCameraPlugin)
             .add_before::<OpenXrPlugin, _>(XrEarlyInitPlugin)
-            // .add(HandPlugin)
+            .add(HandPlugin)
-            // .add(HandTrackingPlugin)
+            .add(HandTrackingPlugin)
-            // .add(HandEmulationPlugin)
+            .add(HandEmulationPlugin)
-            // .add(PassthroughPlugin)
+            .add(PassthroughPlugin)
             .add(XrResourcePlugin)
             .set(WindowPlugin {
                 #[cfg(not(target_os = "android"))]
@@ -369,6 +372,7 @@ pub fn xr_wait_frame(
         **world.get_resource_mut::<XrShouldRender>().unwrap() = should_render;
         **world.get_resource_mut::<XrHasWaited>().unwrap() = true;
     }
+    world.get_resource::<XrSwapchain>().unwrap().begin().unwrap();
 }
 
 pub fn xr_pre_frame(
@@ -458,11 +462,26 @@ pub fn locate_views(
         xr_frame_state.predicted_display_time,
         &input.stage,
     ) {
-        Ok(this) => this,
+        Ok(this) => this
+            .1
+            .into_iter()
+            .map(|mut view| {
+                use crate::prelude::*;
+                let quat = view.pose.orientation.to_quat();
+                let fixed_quat = verify_quat(quat);
+                let oxr_quat = xr::Quaternionf {
+                    x: fixed_quat.x,
+                    y: fixed_quat.y,
+                    z: fixed_quat.z,
+                    w: fixed_quat.w,
+                };
+                view.pose.orientation = oxr_quat;
+                view
+            })
+            .collect(),
         Err(err) => {
             warn!("error: {}", err);
             return;
         }
     }
-    .1;
 }

src/resources.rs

@@ -134,6 +134,14 @@ pub struct SwapchainInner<G: xr::Graphics> {
     pub(crate) buffers: Vec<wgpu::Texture>,
     pub(crate) image_index: Mutex<usize>,
 }
+impl<G: xr::Graphics> Drop for SwapchainInner<G> {
+    fn drop(&mut self) {
+        for _ in 0..self.buffers.len() {
+            let v = self.buffers.remove(0);
+            Box::leak(Box::new(v));
+        }
+    }
+}
 
 impl<G: xr::Graphics> SwapchainInner<G> {
     fn begin(&self) -> xr::Result<()> {
@@ -204,9 +212,7 @@ impl<G: xr::Graphics> SwapchainInner<G> {
                     predicted_display_time,
                     environment_blend_mode,
                     &[
-                        &CompositionLayerPassthrough::from_xr_passthrough_layer(
+                        &CompositionLayerPassthrough::from_xr_passthrough_layer(pass),
-                            pass,
-                        ),
                         &xr::CompositionLayerProjection::new()
                             .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
                             .space(stage)

src/xr_init/mod.rs

@@ -83,6 +83,7 @@ impl Plugin for XrInitPlugin {
         );
         app.add_systems(XrSetup, setup_manual_texture_views);
         app.add_systems(XrCleanup, set_cleanup_res);
+        app.add_systems(PreUpdate, remove_cleanup_res.before(cleanup_xr));
         let render_app = app.sub_app_mut(RenderApp);
         render_app.add_systems(
             Render,
@@ -220,5 +221,5 @@ fn stop_xr_session(session: ResMut<XrSession>, mut status: ResMut<XrStatus>) {
             error!("Error while trying to request session exit: {}", err)
         }
     }
-    *status = XrStatus::Enabling;
+    *status = XrStatus::Disabling;
 }

src/xr_input/mod.rs

@@ -31,7 +31,7 @@ use bevy::utils::HashMap;
 use openxr::Binding;
 
 use self::actions::{setup_oxr_actions, XrActionsPlugin};
-use self::oculus_touch::{init_subaction_path, post_action_setup_oculus_controller, ActionSets};
+use self::oculus_touch::{init_subaction_path, post_action_setup_oculus_controller, ActionSets, OculusController};
 use self::trackers::{
     adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye,
     OpenXRTrackingRoot,
@@ -39,17 +39,18 @@ use self::trackers::{
 use self::xr_camera::{/* GlobalTransformExtract, TransformExtract, */ XrCamera};
 
 #[derive(Copy, Clone)]
-pub struct XrInput;
+pub struct XrInputPlugin;
 #[derive(Clone, Copy, Debug, Ord, PartialOrd, Eq, PartialEq, Component)]
 pub enum Hand {
     Left,
     Right,
 }
 
-impl Plugin for XrInput {
+impl Plugin for XrInputPlugin {
     fn build(&self, app: &mut App) {
         app.add_systems(XrPostSetup, post_action_setup_oculus_controller);
         app.add_systems(XrSetup, setup_oculus_controller);
+        app.add_systems(XrCleanup, cleanup_oculus_controller);
         //adopt any new trackers
         app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only()));
         // app.add_systems(PreUpdate, action_set_system.run_if(xr_only()));
@@ -61,6 +62,10 @@ impl Plugin for XrInput {
     }
 }
 
+fn cleanup_oculus_controller(mut commands: Commands) {
+commands.remove_resource::<OculusController>();
+}
+
 fn cleanup_xr_root(
     mut commands: Commands,
     tracking_root_query: Query<Entity, With<OpenXRTrackingRoot>>,
@@ -74,6 +79,7 @@ fn setup_xr_root(
     tracking_root_query: Query<Entity, With<OpenXRTrackingRoot>>,
 ) {
     if tracking_root_query.get_single().is_err() {
+        info!("Creating XrTrackingRoot!");
         commands.spawn((SpatialBundle::default(), OpenXRTrackingRoot));
     }
 }

src/xr_input/xr_camera.rs

@@ -1,23 +1,27 @@
+use crate::prelude::XrSystems;
 use crate::xr_init::{xr_only, XrCleanup, XrSetup};
 use crate::xr_input::{QuatConv, Vec3Conv};
 use crate::{locate_views, xr_wait_frame, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
 use bevy::core_pipeline::core_3d::graph::Core3d;
 use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping};
+use bevy::ecs::system::lifetimeless::Read;
 use bevy::math::Vec3A;
 use bevy::prelude::*;
 use bevy::render::camera::{
-    CameraMainTextureUsages, CameraProjection, CameraProjectionPlugin, CameraRenderGraph, RenderTarget
+    CameraMainTextureUsages, CameraProjection, CameraProjectionPlugin, CameraRenderGraph,
+    RenderTarget,
 };
 use bevy::render::extract_component::{ExtractComponent, ExtractComponentPlugin};
 use bevy::render::primitives::Frustum;
 use bevy::render::view::{
-    update_frusta, ColorGrading, VisibilitySystems, VisibleEntities,
+    update_frusta, ColorGrading, ExtractedView, VisibilitySystems, VisibleEntities,
 };
+use bevy::render::{Render, RenderApp, RenderSet};
 use bevy::transform::TransformSystem;
 use openxr::Fovf;
 use wgpu::TextureUsages;
 
-use super::trackers::{OpenXRLeftEye, OpenXRRightEye, OpenXRTracker};
+use super::trackers::{OpenXRLeftEye, OpenXRRightEye, OpenXRTracker, OpenXRTrackingRoot};
 
 pub struct XrCameraPlugin;
 
@@ -44,12 +48,29 @@ impl Plugin for XrCameraPlugin {
                 .after(TransformSystem::TransformPropagate)
                 .before(VisibilitySystems::UpdatePerspectiveFrusta),
         );
+        app.add_systems(
+            PostUpdate,
+            update_root_transform_components
+                .after(TransformSystem::TransformPropagate)
+                .xr_only(),
+        );
         app.add_systems(XrSetup, setup_xr_cameras);
         app.add_systems(XrCleanup, cleanup_xr_cameras);
         app.add_plugins(ExtractComponentPlugin::<XrCamera>::default());
         app.add_plugins(ExtractComponentPlugin::<XRProjection>::default());
+        app.add_plugins(ExtractComponentPlugin::<RootTransform>::default());
         // app.add_plugins(ExtractComponentPlugin::<TransformExtract>::default());
         // app.add_plugins(ExtractComponentPlugin::<GlobalTransformExtract>::default());
+        let render_app = app.sub_app_mut(RenderApp);
+        render_app.add_systems(
+            Render,
+            (locate_views, xr_camera_head_sync_render_world)
+                .chain()
+                .run_if(xr_only())
+                .in_set(RenderSet::PrepareAssets),
+            // .after(xr_wait_frame)
+            // .after(locate_views),
+        );
     }
 }
 
@@ -103,10 +124,30 @@ pub struct XrCameraBundle {
     pub color_grading: ColorGrading,
     pub main_texture_usages: CameraMainTextureUsages,
     pub xr_camera_type: XrCamera,
+    pub root_transform: RootTransform,
 }
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)]
 pub struct XrCamera(Eye);
 
+#[derive(Component, ExtractComponent, Clone, Copy, Debug, Default, Deref, DerefMut)]
+pub struct RootTransform(pub GlobalTransform);
+
+fn update_root_transform_components(
+    mut component_query: Query<&mut RootTransform>,
+    root_query: Query<&GlobalTransform, With<OpenXRTrackingRoot>>,
+) {
+    let root = match root_query.get_single() {
+        Ok(v) => v,
+        Err(err) => {
+            warn!("No or too many XrTracking Roots: {}", err);
+            return;
+        }
+    };
+    component_query
+        .par_iter_mut()
+        .for_each(|mut root_transform| **root_transform = *root);
+}
+
 // #[derive(Component)]
 // pub(super) struct TransformExtract;
 //
@@ -171,6 +212,7 @@ impl XrCameraBundle {
                     | TextureUsages::TEXTURE_BINDING
                     | TextureUsages::COPY_SRC,
             ),
+            root_transform: default(),
         }
     }
 }
@@ -349,3 +391,20 @@ pub fn xr_camera_head_sync(
         transform.translation = view.pose.position.to_vec3();
     }
 }
+
+pub fn xr_camera_head_sync_render_world(
+    views: Res<crate::resources::XrViews>,
+    mut query: Query<(&mut ExtractedView, &XrCamera, &RootTransform)>,
+) {
+    for (mut extracted_view, camera_type, root) in query.iter_mut() {
+        let view_idx = camera_type.0 as usize;
+        let view = match views.get(view_idx) {
+            Some(views) => views,
+            None => continue,
+        };
+        let mut transform = Transform::IDENTITY;
+        transform.rotation = view.pose.orientation.to_quat();
+        transform.translation = view.pose.position.to_vec3();
+        extracted_view.transform = root.mul_transform(transform);
+    }
+}