Merge pull request #68 from Schmarni-Dev/runtime_sessions

Allow XR Session starting and stopping at runtime, add pipelined rendering and fix on platforms without fb passthrough

Cargo.toml

@@ -15,14 +15,15 @@ force-link = ["openxr/linked"]
 members = ["examples/android", "examples/demo"]
 
 [dependencies]
-anyhow = "1.0.75"
+# anyhow = "1.0.75"
 ash = "0.37.3"
-bevy = "0.12"
+bevy = "0.13"
 futures-lite = "2.0.1"
 mint = "0.5.9"
-wgpu = "0.17.1"
-wgpu-core = { version = "0.17.1", features = ["vulkan"] }
-wgpu-hal = "0.17.1"
+wgpu = "0.19"
+wgpu-core = { version = "0.19", features = ["vulkan"] }
+wgpu-hal = "0.19"
+eyre = "0.6.11"
 
 [target.'cfg(windows)'.dependencies]
 openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = [
@@ -45,9 +46,9 @@ ndk-context = "0.1"
 jni = "0.20"
 
 [dev-dependencies]
-bevy = "0.12"
-color-eyre = "0.6.2"
+bevy = "0.13"
 bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" }
+color-eyre = "0.6.2"
 
 [[example]]
 name = "xr"
@@ -57,4 +58,4 @@ path = "examples/xr.rs"
 debug = true
 
 [patch.crates-io]
-ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" }
+# ndk = { git = "https://github.com/Schmarni-Dev/ndk.git", branch = "070" }

examples/android/Cargo.toml

@@ -13,13 +13,13 @@ crate-type = ["rlib", "cdylib"]
 
 [dependencies]
 bevy_oxr.path = "../.."
-bevy = "0.12"
+bevy = "0.13"
 openxr = { git = "https://github.com/Ralith/openxrs", rev = "0177d2d", features = ["mint"] }
 
-[profile.release]
-lto = "fat"
-codegen-units = 1
-panic = "abort"
+# [profile.release]
+# lto = "fat"
+# codegen-units = 1
+# panic = "abort"
 
 # This metadata is used by `cargo-apk` - `xbuild` uses the `manifest.yaml` instead.
 [package.metadata.android]

examples/android/manifest.yaml

@@ -12,6 +12,8 @@ android:
         required: true
       - name: "com.oculus.experimental.enabled"
         required: true
+    uses_permission:
+      - name: "com.oculus.permission.HAND_TRACKING"
     application:
       label: "Bevy Openxr Android"
       theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen"

examples/android/src/lib.rs

@@ -3,10 +3,11 @@ use bevy::prelude::*;
 use bevy::transform::components::Transform;
 use bevy_oxr::graphics::extensions::XrExtensions;
 use bevy_oxr::graphics::XrAppInfo;
-use bevy_oxr::graphics::XrPreferdBlendMode::AlphaBlend;
-use bevy_oxr::passthrough::{passthrough_layer_pause, passthrough_layer_resume};
-use bevy_oxr::xr_init::XrRenderData;
+use bevy_oxr::passthrough::{PausePassthrough, ResumePassthrough, XrPassthroughState};
+use bevy_oxr::xr_init::xr_only;
 use bevy_oxr::xr_input::debug_gizmos::OpenXrDebugRenderer;
+use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer;
+use bevy_oxr::xr_input::hands::HandBone;
 use bevy_oxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig};
 use bevy_oxr::xr_input::trackers::{
     OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker,
@@ -16,7 +17,8 @@ use bevy_oxr::DefaultXrPlugins;
 #[bevy_main]
 fn main() {
     let mut xr_extensions = XrExtensions::default();
-    xr_extensions.enable_fb_passthrough();
+    // xr_extensions.enable_fb_passthrough();
+    xr_extensions.enable_hand_tracking();
     App::new()
         .add_plugins(DefaultXrPlugins {
             reqeusted_extensions: xr_extensions,
@@ -25,16 +27,28 @@ fn main() {
             },
             prefered_blend_mode: bevy_oxr::graphics::XrPreferdBlendMode::Opaque,
         })
-        .add_plugins(OpenXrDebugRenderer)
+        // .add_plugins(OpenXrDebugRenderer)
         .add_plugins(LogDiagnosticsPlugin::default())
         .add_plugins(FrameTimeDiagnosticsPlugin)
+        .add_plugins(HandInputDebugRenderer)
+        // .add_plugins(bevy_oxr::passthrough::EnablePassthroughStartup)
         .add_systems(Startup, setup)
-        .add_systems(Update, (proto_locomotion, toggle_passthrough))
+        .add_systems(
+            Update,
+            (proto_locomotion, toggle_passthrough).run_if(xr_only()),
+        )
+        .add_systems(Update, debug_hand_render.run_if(xr_only()))
         .add_systems(Startup, spawn_controllers_example)
         .insert_resource(PrototypeLocomotionConfig::default())
         .run();
 }
 
+fn debug_hand_render(query: Query<&GlobalTransform, With<HandBone>>, mut gizmos: Gizmos) {
+    for transform in &query {
+        gizmos.sphere(transform.translation(), Quat::IDENTITY, 0.01, Color::RED);
+    }
+}
+
 /// set up a simple 3D scene
 fn setup(
     mut commands: Commands,
@@ -43,21 +57,21 @@ fn setup(
 ) {
     // plane
     commands.spawn(PbrBundle {
-        mesh: meshes.add(shape::Plane::from_size(5.0).into()),
-        material: materials.add(Color::rgb(0.3, 0.5, 0.3).into()),
+        mesh: meshes.add(Plane3d::new(Vec3::Y)),
+        material: materials.add(StandardMaterial::from(Color::rgb(0.3, 0.5, 0.3))),
         ..default()
     });
     // cube
     commands.spawn(PbrBundle {
-        mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
-        material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+        mesh: meshes.add(Cuboid::from_size(Vec3::splat(0.1)).mesh()),
+        material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))),
         transform: Transform::from_xyz(0.0, 0.5, 0.0),
         ..default()
     });
     // cube
     commands.spawn(PbrBundle {
-        mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
-        material: materials.add(Color::rgb(0.8, 0.0, 0.0).into()),
+        mesh: meshes.add(Mesh::from(Cuboid::from_size(Vec3::splat(0.1)))),
+        material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.0, 0.0))),
         transform: Transform::from_xyz(0.0, 0.5, 1.0),
         ..default()
     });
@@ -90,15 +104,22 @@ fn spawn_controllers_example(mut commands: Commands) {
     ));
 }
 
-// Does this work? Not getting logs
-fn toggle_passthrough(keys: Res<Input<KeyCode>>, mut xr_data: ResMut<XrRenderData>) {
+// TODO: make this a vr button
+fn toggle_passthrough(
+    keys: Res<ButtonInput<KeyCode>>,
+    passthrough_state: Res<XrPassthroughState>,
+    mut resume: EventWriter<ResumePassthrough>,
+    mut pause: EventWriter<PausePassthrough>,
+) {
     if keys.just_pressed(KeyCode::Space) {
-        if xr_data.xr_passthrough_active {
-            passthrough_layer_pause(xr_data);
-            bevy::log::info!("Passthrough paused");
-        } else {
-            passthrough_layer_resume(xr_data);
-            bevy::log::info!("Passthrough resumed");
+        match *passthrough_state {
+            XrPassthroughState::Unsupported => {}
+            XrPassthroughState::Running => {
+                pause.send_default();
+            }
+            XrPassthroughState::Paused => {
+                resume.send_default();
+            }
         }
     }
 }

examples/demo/Cargo.toml

@@ -9,9 +9,10 @@ crate-type = ["rlib", "cdylib"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bevy = "0.12"
+bevy = "0.13"
 bevy_oxr.path = "../../"
-bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" }
+# bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" }
+bevy_rapier3d = "0.25"
 color-eyre = "0.6.2"
 
 

examples/demo/manifest.yaml

@@ -3,26 +3,29 @@ android:
     - "runtime_libs"
   manifest:
     package: "org.bevyengine.demo_openxr_android"
-    # Are features and permissions fliped?
     uses_feature:
       - name: "android.hardware.vr.headtracking"
         required: true
       - name: "oculus.software.handtracking"
-        required: false
-      # - name: "com.oculus.feature.PASSTHROUGH"
-      #   required: true
+        required: true
+      - name: "com.oculus.feature.PASSTHROUGH"
+        required: true
+      - name: "com.oculus.experimental.enabled"
+        required: true
     uses_permission:
       - name: "com.oculus.permission.HAND_TRACKING"
     application:
       label: "Bevy Openxr Android"
       theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen"
       meta_data:
+        - name: "com.oculus.intent.category.VR"
+          value: "vr_only"
         - name: "com.samsung.android.vr.application.mode"
           value: "vr_only"
         - name: "com.oculus.supportedDevices"
-          value: "quest|quest2|quest3"
+          value: "quest|quest2|quest3|questpro"
       activities:
-        - config_changes: "density|keyboard|keyboardHidden|navigation|orientation|screenLayout|screenSize|uiMode"
+        - config_changes: "density|keyboard|keyboardHidden|navigation|orientation|screenLayout|screenSize|uiMode|screenLayout"
           launch_mode: "singleTask"
           orientation: "landscape"
           intent_filters:
@@ -31,5 +34,6 @@ android:
               categories:
                 - "com.oculus.intent.category.VR"
                 - "android.intent.category.LAUNCHER"
+                - "org.khronos.openxr.intent.category.IMMERSIVE_HMD"
     sdk:
       target_sdk_version: 32

examples/demo/src/lib.rs

@@ -3,8 +3,9 @@ use std::{f32::consts::PI, ops::Mul, time::Duration};
 use bevy::{
     diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin},
     ecs::schedule::ScheduleLabel,
-    input::{keyboard::KeyCode, Input},
+    input::{keyboard::KeyCode, ButtonInput},
     log::info,
+    math::primitives::{Capsule3d, Cuboid},
     prelude::{
         bevy_main, default, shape, App, Assets, Color, Commands, Component, Entity, Event,
         EventReader, EventWriter, FixedUpdate, Gizmos, GlobalTransform, IntoSystemConfigs,
@@ -12,6 +13,7 @@ use bevy::{
         Schedule, SpatialBundle, StandardMaterial, Startup, Transform, Update, Vec3, Vec3Swizzles,
         With, Without, World,
     },
+    render::mesh::Meshable,
     time::{Fixed, Time, Timer, TimerMode},
     transform::TransformSystem,
 };
@@ -19,11 +21,11 @@ use bevy_oxr::{
     graphics::{extensions::XrExtensions, XrAppInfo, XrPreferdBlendMode},
     input::XrInput,
     resources::{XrFrameState, XrInstance, XrSession},
-    xr_init::{xr_only, XrEnableRequest, XrEnableStatus},
+    xr_init::{xr_only, XrStatus},
     xr_input::{
         actions::XrActionSets,
         debug_gizmos::OpenXrDebugRenderer,
-        hands::common::{HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
+        hands::common::{HandInputDebugRenderer, HandResource, HandsResource},
         hands::HandBone,
         interactions::{
             draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
@@ -41,19 +43,18 @@ use bevy_oxr::{
     DefaultXrPlugins,
 };
 
-fn input_stuff(
-    keys: Res<Input<KeyCode>>,
-    status: Res<XrEnableStatus>,
-    mut request: EventWriter<XrEnableRequest>,
-) {
-    if keys.just_pressed(KeyCode::Space) {
-        match status.into_inner() {
-            XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable),
-            XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable),
-            XrEnableStatus::Waiting => (),
-        }
-    }
-}
+// fn input_stuff(
+//     keys: Res<Input<KeyCode>>,
+//     status: Res<XrEnableStatus>,
+//     mut request: EventWriter<XrEnableRequest>,
+// ) {
+//     if keys.just_pressed(KeyCode::Space) {
+//         match status.into_inner() {
+//             XrEnableStatus::Enabled => request.send(XrEnableRequest::TryDisable),
+//             XrEnableStatus::Disabled => request.send(XrEnableRequest::TryEnable),
+//         }
+//     }
+// }
 
 mod setup;
 use crate::setup::setup_scene;
@@ -66,8 +67,9 @@ pub fn main() {
     info!("Running bevy_openxr demo");
     let mut app = App::new();
     let mut xr_extensions = XrExtensions::default();
+    xr_extensions.enable_fb_passthrough();
 
-    app.add_systems(Update, input_stuff)
+    app
         //lets get the usual diagnostic stuff added
         .add_plugins(LogDiagnosticsPlugin::default())
         .add_plugins(FrameTimeDiagnosticsPlugin)
@@ -120,7 +122,7 @@ pub fn main() {
         //test capsule
         .add_systems(Startup, spawn_capsule)
         //physics hands
-        .add_plugins(OpenXrHandInput)
+        // .add_plugins(OpenXrHandInput)
         .add_plugins(HandInputDebugRenderer)
         .add_systems(Startup, spawn_physics_hands)
         .add_systems(
@@ -226,12 +228,8 @@ fn spawn_capsule(
 ) {
     commands.spawn((
         PbrBundle {
-            mesh: meshes.add(Mesh::from(shape::Capsule {
-                radius: 0.033,
-                depth: 0.115,
-                ..default()
-            })),
-            material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+            mesh: meshes.add(Capsule3d::new(0.033, 0.115).mesh()),
+            material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))),
             transform: Transform::from_xyz(0.0, 2.0, 0.0),
             ..default()
         },
@@ -376,7 +374,7 @@ fn update_physics_hands(
         &Hand,
         &mut Velocity,
     )>,
-    hand_query: Query<(&Transform, &HandBone, &Hand, Without<PhysicsHandBone>)>,
+    hand_query: Query<(&Transform, &HandBone, &Hand), Without<PhysicsHandBone>>,
     time: Res<Time>,
     mut gizmos: Gizmos,
 ) {
@@ -560,8 +558,6 @@ fn request_cube_spawn(
 ) {
     timer.0.tick(time.delta());
     if timer.0.finished() {
-        //lock frame
-        let frame_state = *frame_state.lock().unwrap();
         //get controller
         let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
         //get controller triggers
@@ -588,8 +584,8 @@ fn cube_spawner(
         // cube
         commands.spawn((
             PbrBundle {
-                mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
-                material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+                mesh: meshes.add(Cuboid::from_size(Vec3::splat(0.1)).mesh()),
+                material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))),
                 transform: Transform::from_xyz(0.0, 1.0, 0.0),
                 ..default()
             },
@@ -628,8 +624,6 @@ fn prototype_interaction_input(
     >,
     action_sets: Res<XrActionSets>,
 ) {
-    //lock frame
-    let frame_state = *frame_state.lock().unwrap();
     //get controller
     let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
     //get controller triggers
@@ -664,7 +658,7 @@ pub struct GhostTimers {
 
 pub fn handle_ghost_hand_events(
     mut events: EventReader<GhostHandEvent>,
-    mut bones: Query<(&Hand, &mut CollisionGroups, With<PhysicsHandBone>)>,
+    mut bones: Query<(&Hand, &mut CollisionGroups), With<PhysicsHandBone>>,
 ) {
     for event in events.read() {
         // info!(
@@ -712,20 +706,19 @@ pub struct Grabbable;
 
 pub fn update_grabbables(
     mut events: EventReader<InteractionEvent>,
-    mut grabbable_query: Query<(
-        Entity,
-        &mut Transform,
-        With<Grabbable>,
-        Without<XRDirectInteractor>,
-        Option<&mut RigidBody>,
-    )>,
-    mut interactor_query: Query<(
+    mut grabbable_query: Query<
+        (Entity, &mut Transform, Option<&mut RigidBody>),
+        (Without<XRDirectInteractor>, With<Grabbable>),
+    >,
+    mut interactor_query: Query<
+        (
             &GlobalTransform,
             &XRInteractorState,
             &mut XRSelection,
             &Hand,
+        ),
         Without<Grabbable>,
-    )>,
+    >,
     mut writer: EventWriter<GhostHandEvent>,
     mut timers: ResMut<GhostTimers>,
 ) {
@@ -741,7 +734,7 @@ pub fn update_grabbables(
                         match *interactor_transform.2 {
                             XRSelection::Empty => {
                                 match interactor_transform.1 {
-                                    XRInteractorState::Idle => match grabbable_transform.4 {
+                                    XRInteractorState::Idle => match grabbable_transform.2 {
                                         Some(mut thing) => {
                                             *thing = RigidBody::Dynamic;
                                             *interactor_transform.2 = XRSelection::Empty;
@@ -750,7 +743,7 @@ pub fn update_grabbables(
                                     },
                                     XRInteractorState::Selecting => {
                                         // info!("its a direct interactor?");
-                                        match grabbable_transform.4 {
+                                        match grabbable_transform.2 {
                                             Some(mut thing) => {
                                                 *thing = RigidBody::KinematicPositionBased;
                                                 *interactor_transform.2 =

examples/demo/src/setup.rs

@@ -1,9 +1,10 @@
 use bevy::{
+    math::primitives::{Cuboid, Plane3d},
     prelude::{
-        shape, Assets, Camera3dBundle, Color, Commands, Mesh, PbrBundle, PointLight,
-        PointLightBundle, ResMut, SpatialBundle, StandardMaterial, Transform, Vec3,
+        Assets, Camera3dBundle, Color, Commands, Mesh, PbrBundle, ResMut, StandardMaterial,
+        Transform, Vec3,
     },
-    transform::TransformBundle,
+    render::mesh::Meshable,
     utils::default,
 };
 use bevy_oxr::xr_input::interactions::{Touched, XRInteractable, XRInteractableState};
@@ -29,8 +30,8 @@ pub fn setup_scene(
     // plane
     commands.spawn((
         PbrBundle {
-            mesh: meshes.add(shape::Plane::from_size(5.0).into()),
-            material: materials.add(Color::rgb(0.3, 0.5, 0.3).into()),
+            mesh: meshes.add(Plane3d::new(Vec3::Y).mesh()),
+            material: materials.add(StandardMaterial::from(Color::rgb(0.3, 0.5, 0.3))),
             transform: Transform::from_xyz(0.0, ground_height, 0.0),
             ..default()
         },
@@ -41,8 +42,8 @@ pub fn setup_scene(
     // cube
     commands.spawn((
         PbrBundle {
-            mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
-            material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+            mesh: meshes.add(Cuboid::from_size(Vec3::splat(0.1))),
+            material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))),
             transform: Transform::from_xyz(0.0, 1.0, 0.0),
             ..default()
         },

examples/globe.rs

@@ -1,9 +1,10 @@
 use bevy::diagnostic::LogDiagnosticsPlugin;
 use bevy::prelude::*;
+use bevy::render::render_asset::RenderAssetUsages;
 use bevy::transform::components::Transform;
 use bevy_oxr::graphics::XrAppInfo;
 use bevy_oxr::resources::XrViews;
-use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput};
+use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer;
 use bevy_oxr::xr_input::interactions::{
     InteractionEvent, XRDirectInteractor, XRInteractorState, XRRayInteractor, XRSocketInteractor,
 };
@@ -31,7 +32,6 @@ fn main() {
         .add_systems(Update, (proto_locomotion, pull_to_ground).chain())
         .insert_resource(PrototypeLocomotionConfig::default())
         .add_systems(Startup, spawn_controllers_example)
-        .add_plugins(OpenXrHandInput)
         .add_plugins(HandInputDebugRenderer)
         .add_event::<InteractionEvent>()
         .run();
@@ -67,6 +67,7 @@ fn uv_debug_texture() -> Image {
         TextureDimension::D2,
         &texture_data,
         TextureFormat::Rgba8UnormSrgb,
+        RenderAssetUsages::RENDER_WORLD,
     )
 }
 
@@ -81,15 +82,7 @@ fn setup(
     let radius = 5.0;
     commands.spawn((
         PbrBundle {
-            mesh: meshes.add(
-                shape::UVSphere {
-                    radius,
-                    sectors: 10,
-                    stacks: 10,
-                }
-                .try_into()
-                .unwrap(),
-            ),
+            mesh: meshes.add(Sphere::new(radius)),
             material: materials.add(StandardMaterial {
                 base_color_texture: Some(images.add(uv_debug_texture())),
                 ..default()
@@ -101,8 +94,8 @@ fn setup(
     ));
     // cube
     commands.spawn(PbrBundle {
-        mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
-        material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+        mesh: meshes.add(Cuboid::from_size(Vec3::splat(0.1)).mesh()),
+        material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))),
         transform: Transform::from_xyz(0.0, 0.5, 0.0),
         ..default()
     });
@@ -172,8 +165,7 @@ fn pull_to_ground(
     let (globe_pos, globe) = globe.single();
 
     // Get player position (position of playground + position within playground)
-    let v = views.lock().unwrap();
-    let Some(view) = v.get(0) else { return };
+    let Some(view) = views.first() else { return };
     let mut hmd_translation = view.pose.position.to_vec3();
     hmd_translation.y = 0.0;
     let local = root.translation;
@@ -188,7 +180,7 @@ fn pull_to_ground(
     root.translation += diff * adjustment_rate;
 
     // Rotate player to be upright on sphere
-    let angle_diff = Quat::from_rotation_arc(root.up(), up);
+    let angle_diff = Quat::from_rotation_arc(*root.up(), up);
     let point = root.translation + offset;
     root.rotate_around(point, Quat::IDENTITY.slerp(angle_diff, adjustment_rate));
 }

examples/xr.rs

@@ -6,8 +6,9 @@ use bevy_oxr::graphics::XrAppInfo;
 use bevy_oxr::input::XrInput;
 use bevy_oxr::resources::{XrFrameState, XrSession};
 
+use bevy_oxr::xr_init::{xr_only, EndXrSession, StartXrSession, XrSetup};
 use bevy_oxr::xr_input::actions::XrActionSets;
-use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput};
+use bevy_oxr::xr_input::hands::common::HandInputDebugRenderer;
 use bevy_oxr::xr_input::interactions::{
     draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
     update_interactable_states, InteractionEvent, Touched, XRDirectInteractor, XRInteractable,
@@ -36,28 +37,53 @@ fn main() {
         .add_plugins(LogDiagnosticsPlugin::default())
         .add_plugins(FrameTimeDiagnosticsPlugin)
         .add_systems(Startup, setup)
-        .add_systems(Update, proto_locomotion)
+        .add_systems(Update, proto_locomotion.run_if(xr_only()))
         .insert_resource(PrototypeLocomotionConfig::default())
-        .add_systems(Startup, spawn_controllers_example)
-        .add_plugins(OpenXrHandInput)
+        .add_systems(XrSetup, spawn_controllers_example)
         .add_plugins(HandInputDebugRenderer)
         .add_systems(
             Update,
-            draw_interaction_gizmos.after(update_interactable_states),
+            draw_interaction_gizmos
+                .after(update_interactable_states)
+                .run_if(xr_only()),
+        )
+        .add_systems(
+            Update,
+            draw_socket_gizmos
+                .after(update_interactable_states)
+                .run_if(xr_only()),
+        )
+        .add_systems(
+            Update,
+            interactions
+                .before(update_interactable_states)
+                .run_if(xr_only()),
         )
-        .add_systems(Update, draw_socket_gizmos.after(update_interactable_states))
-        .add_systems(Update, interactions.before(update_interactable_states))
         .add_systems(
             Update,
             socket_interactions.before(update_interactable_states),
         )
-        .add_systems(Update, prototype_interaction_input)
+        .add_systems(Update, prototype_interaction_input.run_if(xr_only()))
         .add_systems(Update, update_interactable_states)
         .add_systems(Update, update_grabbables.after(update_interactable_states))
+        .add_systems(Update, start_stop_session)
         .add_event::<InteractionEvent>()
         .run();
 }
 
+fn start_stop_session(
+    keyboard: Res<ButtonInput<KeyCode>>,
+    mut start: EventWriter<StartXrSession>,
+    mut stop: EventWriter<EndXrSession>,
+) {
+    if keyboard.just_pressed(KeyCode::KeyS) {
+        start.send_default();
+    }
+    if keyboard.just_pressed(KeyCode::KeyE) {
+        stop.send_default();
+    }
+}
+
 /// set up a simple 3D scene
 fn setup(
     mut commands: Commands,
@@ -66,14 +92,14 @@ fn setup(
 ) {
     // plane
     commands.spawn(PbrBundle {
-        mesh: meshes.add(shape::Plane::from_size(5.0).into()),
-        material: materials.add(Color::rgb(0.3, 0.5, 0.3).into()),
+        mesh: meshes.add(Plane3d::new(Vec3::Y).mesh()),
+        material: materials.add(StandardMaterial::from(Color::rgb(0.3, 0.5, 0.3))),
         ..default()
     });
     // cube
     commands.spawn(PbrBundle {
-        mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
-        material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+        mesh: meshes.add(Cuboid::from_size(Vec3::splat(0.1)).mesh()),
+        material: materials.add(StandardMaterial::from(Color::rgb(0.8, 0.7, 0.6))),
         transform: Transform::from_xyz(0.0, 0.5, 0.0),
         ..default()
     });
@@ -144,13 +170,14 @@ fn spawn_controllers_example(mut commands: Commands) {
     ));
 }
 
+#[allow(clippy::type_complexity)]
 fn prototype_interaction_input(
     oculus_controller: Res<OculusController>,
     frame_state: Res<XrFrameState>,
     xr_input: Res<XrInput>,
     session: Res<XrSession>,
     mut right_interactor_query: Query<
-        (&mut XRInteractorState),
+        &mut XRInteractorState,
         (
             With<XRDirectInteractor>,
             With<OpenXRRightController>,
@@ -158,7 +185,7 @@ fn prototype_interaction_input(
         ),
     >,
     mut left_interactor_query: Query<
-        (&mut XRInteractorState),
+        &mut XRInteractorState,
         (
             With<XRRayInteractor>,
             With<OpenXRLeftController>,
@@ -167,8 +194,6 @@ fn prototype_interaction_input(
     >,
     action_sets: Res<XrActionSets>,
 ) {
-    //lock frame
-    let frame_state = *frame_state.lock().unwrap();
     //get controller
     let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
     //get controller triggers
@@ -194,8 +219,8 @@ pub struct Grabbable;
 
 pub fn update_grabbables(
     mut events: EventReader<InteractionEvent>,
-    mut grabbable_query: Query<(&mut Transform, With<Grabbable>, Without<XRDirectInteractor>)>,
-    interactor_query: Query<(&GlobalTransform, &XRInteractorState, Without<Grabbable>)>,
+    mut grabbable_query: Query<&mut Transform, (With<Grabbable>, Without<XRDirectInteractor>)>,
+    interactor_query: Query<(&GlobalTransform, &XRInteractorState), Without<Grabbable>>,
 ) {
     //so basically the idea is to try all the events?
     for event in events.read() {
@@ -210,7 +235,7 @@ pub fn update_grabbables(
                             XRInteractorState::Idle => (),
                             XRInteractorState::Selecting => {
                                 // info!("its a direct interactor?");
-                                *grabbable_transform.0 = interactor_transform.0.compute_transform();
+                                *grabbable_transform = interactor_transform.0.compute_transform();
                             }
                         }
                     }

src/graphics/mod.rs

@@ -1,16 +1,21 @@
 pub mod extensions;
 mod vulkan;
 
-use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue};
-use bevy::window::RawHandleWrapper;
+use bevy::ecs::query::With;
+use bevy::ecs::system::{Query, SystemState};
+use bevy::ecs::world::World;
+use bevy::render::renderer::{
+    RenderAdapter, RenderAdapterInfo, RenderDevice, RenderInstance, RenderQueue,
+};
+use bevy::window::{PrimaryWindow, RawHandleWrapper};
 use wgpu::Instance;
 
 use crate::input::XrInput;
-use crate::passthrough::{Passthrough, PassthroughLayer};
 use crate::resources::{
-    XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrPassthrough,
-    XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews,
+    XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution,
+    XrSession, XrSessionRunning, XrSwapchain, XrViews,
 };
+use crate::OXrSessionSetupInfo;
 
 use openxr as xr;
 
@@ -40,20 +45,15 @@ impl Default for XrAppInfo {
     }
 }
 
-pub fn initialize_xr_graphics(
+pub fn start_xr_session(
     window: Option<RawHandleWrapper>,
-    reqeusted_extensions: XrExtensions,
-    prefered_blend_mode: XrPreferdBlendMode,
-    app_info: XrAppInfo,
-) -> anyhow::Result<(
-    RenderDevice,
-    RenderQueue,
-    RenderAdapterInfo,
-    RenderAdapter,
-    Instance,
-    XrInstance,
+    session_setup_data: &OXrSessionSetupInfo,
+    xr_instance: &XrInstance,
+    render_device: &RenderDevice,
+    render_adapter: &RenderAdapter,
+    wgpu_instance: &Instance,
+) -> eyre::Result<(
     XrSession,
-    XrEnvironmentBlendMode,
     XrResolution,
     XrFormat,
     XrSessionRunning,
@@ -63,13 +63,113 @@ pub fn initialize_xr_graphics(
     XrViews,
     XrFrameState,
 )> {
-    vulkan::initialize_xr_graphics(window, reqeusted_extensions, prefered_blend_mode, app_info)
+    vulkan::start_xr_session(
+        window,
+        session_setup_data,
+        xr_instance,
+        render_device,
+        render_adapter,
+        wgpu_instance,
+    )
+}
+pub fn initialize_xr_instance(
+    window: Option<RawHandleWrapper>,
+    reqeusted_extensions: XrExtensions,
+    prefered_blend_mode: XrPreferdBlendMode,
+    app_info: XrAppInfo,
+) -> eyre::Result<(
+    XrInstance,
+    OXrSessionSetupInfo,
+    XrEnvironmentBlendMode,
+    RenderDevice,
+    RenderQueue,
+    RenderAdapterInfo,
+    RenderAdapter,
+    Instance,
+)> {
+    vulkan::initialize_xr_instance(window, reqeusted_extensions, prefered_blend_mode, app_info)
 }
 
-pub fn xr_entry() -> anyhow::Result<xr::Entry> {
+pub fn try_full_init(
+    world: &mut World,
+    reqeusted_extensions: XrExtensions,
+    prefered_blend_mode: XrPreferdBlendMode,
+    app_info: XrAppInfo,
+) -> eyre::Result<(
+    RenderDevice,
+    RenderQueue,
+    RenderAdapterInfo,
+    RenderAdapter,
+    RenderInstance,
+)> {
+    let mut system_state: SystemState<Query<&RawHandleWrapper, With<PrimaryWindow>>> =
+        SystemState::new(world);
+    let primary_window = system_state.get(world).get_single().ok().cloned();
+    let (
+        xr_instance,
+        setup_info,
+        blend_mode,
+        render_device,
+        render_queue,
+        render_adapter_info,
+        render_adapter,
+        wgpu_instance,
+    ) = initialize_xr_instance(
+        primary_window.clone(),
+        reqeusted_extensions,
+        prefered_blend_mode,
+        app_info,
+    )?;
+    world.insert_resource(xr_instance);
+    world.insert_non_send_resource(setup_info);
+    // TODO: move BlendMode the session init?
+    world.insert_resource(blend_mode);
+    let setup_info = world
+        .get_non_send_resource::<OXrSessionSetupInfo>()
+        .unwrap();
+    let xr_instance = world.get_resource::<XrInstance>().unwrap();
+
+    let (
+        xr_session,
+        xr_resolution,
+        xr_format,
+        xr_session_running,
+        xr_frame_waiter,
+        xr_swapchain,
+        xr_input,
+        xr_views,
+        xr_frame_state,
+    ) = start_xr_session(
+        primary_window,
+        setup_info,
+        xr_instance,
+        &render_device,
+        &render_adapter,
+        &wgpu_instance,
+    )?;
+    world.insert_resource(xr_session);
+    world.insert_resource(xr_resolution);
+    world.insert_resource(xr_format);
+    world.insert_resource(xr_session_running);
+    world.insert_resource(xr_frame_waiter);
+    world.insert_resource(xr_swapchain);
+    world.insert_resource(xr_input);
+    world.insert_resource(xr_views);
+    world.insert_resource(xr_frame_state);
+
+    Ok((
+        render_device,
+        render_queue,
+        render_adapter_info,
+        render_adapter,
+        RenderInstance(wgpu_instance.into()),
+    ))
+}
+
+pub fn xr_entry() -> eyre::Result<xr::Entry> {
     #[cfg(windows)]
     let entry = Ok(xr::Entry::linked());
     #[cfg(not(windows))]
-    let entry = unsafe { xr::Entry::load().map_err(|e| anyhow::anyhow!(e)) };
+    let entry = unsafe { xr::Entry::load().map_err(|e| eyre::eyre!(e)) };
     entry
 }

src/graphics/vulkan.rs

@@ -2,54 +2,45 @@ use std::ffi::{c_void, CString};
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Mutex};
 
-use anyhow::Context;
+// use anyhow::Context;
 use ash::vk::{self, Handle};
 use bevy::math::uvec2;
 use bevy::prelude::*;
 use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue};
 use bevy::window::RawHandleWrapper;
+use eyre::{Context, ContextCompat};
 use openxr as xr;
 use wgpu::Instance;
+use wgpu_hal::{api::Vulkan as V, Api};
 use xr::EnvironmentBlendMode;
 
 use crate::graphics::extensions::XrExtensions;
 use crate::input::XrInput;
 
-use crate::passthrough::{Passthrough, PassthroughLayer};
 use crate::resources::{
-    Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter,
-    XrInstance, XrPassthrough, XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning,
-    XrSwapchain, XrViews,
+    OXrSessionSetupInfo, Swapchain, SwapchainInner, VulkanOXrSessionSetupInfo,
+    XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrResolution,
+    XrSession, XrSessionRunning, XrSwapchain, XrViews,
 };
 use crate::VIEW_TYPE;
 
 use super::{XrAppInfo, XrPreferdBlendMode};
 
-pub fn initialize_xr_graphics(
+pub fn initialize_xr_instance(
     window: Option<RawHandleWrapper>,
     reqeusted_extensions: XrExtensions,
     prefered_blend_mode: XrPreferdBlendMode,
     app_info: XrAppInfo,
-) -> anyhow::Result<(
+) -> eyre::Result<(
+    XrInstance,
+    OXrSessionSetupInfo,
+    XrEnvironmentBlendMode,
     RenderDevice,
     RenderQueue,
     RenderAdapterInfo,
     RenderAdapter,
     Instance,
-    XrInstance,
-    XrSession,
-    XrEnvironmentBlendMode,
-    XrResolution,
-    XrFormat,
-    XrSessionRunning,
-    XrFrameWaiter,
-    XrSwapchain,
-    XrInput,
-    XrViews,
-    XrFrameState,
 )> {
-    use wgpu_hal::{api::Vulkan as V, Api};
-
     let xr_entry = super::xr_entry()?;
 
     #[cfg(target_os = "android")]
@@ -139,9 +130,8 @@ pub fn initialize_xr_graphics(
     }
 
     let vk_entry = unsafe { ash::Entry::load() }?;
-    let flags = wgpu_hal::InstanceFlags::empty();
-    let extensions =
-        <V as Api>::Instance::required_extensions(&vk_entry, vk_target_version, flags)?;
+    let flags = wgpu::InstanceFlags::from_build_config();
+    let extensions = <V as Api>::Instance::desired_extensions(&vk_entry, vk_target_version, flags)?;
     let device_extensions = vec![
         ash::extensions::khr::Swapchain::name(),
         ash::extensions::khr::DrawIndirectCount::name(),
@@ -291,8 +281,8 @@ pub fn initialize_xr_graphics(
             wgpu_open_device,
             &wgpu::DeviceDescriptor {
                 label: None,
-                features: wgpu_features,
-                limits: wgpu::Limits {
+                required_features: wgpu_features,
+                required_limits: wgpu::Limits {
                     max_bind_groups: 8,
                     max_storage_buffer_binding_size: wgpu_adapter
                         .limits()
@@ -304,32 +294,75 @@ pub fn initialize_xr_graphics(
             None,
         )
     }?;
+    Ok((
+        xr_instance.into(),
+        OXrSessionSetupInfo::Vulkan(VulkanOXrSessionSetupInfo {
+            device_ptr: vk_device_ptr,
+            physical_device_ptr: vk_physical_device_ptr,
+            vk_instance_ptr,
+            queue_family_index,
+            xr_system_id,
+        }),
+        blend_mode.into(),
+        wgpu_device.into(),
+        RenderQueue(wgpu_queue.into()),
+        RenderAdapterInfo(wgpu_adapter.get_info()),
+        RenderAdapter(wgpu_adapter.into()),
+        wgpu_instance.into(),
+    ))
+}
 
+pub fn start_xr_session(
+    window: Option<RawHandleWrapper>,
+    ptrs: &OXrSessionSetupInfo,
+    xr_instance: &XrInstance,
+    render_device: &RenderDevice,
+    render_adapter: &RenderAdapter,
+    wgpu_instance: &Instance,
+) -> eyre::Result<(
+    XrSession,
+    XrResolution,
+    XrFormat,
+    XrSessionRunning,
+    XrFrameWaiter,
+    XrSwapchain,
+    XrInput,
+    XrViews,
+    XrFrameState,
+)> {
+    let wgpu_device = render_device.wgpu_device();
+    let wgpu_adapter = &render_adapter.0;
+
+    #[allow(unreachable_patterns)]
+    let setup_info = match ptrs {
+        OXrSessionSetupInfo::Vulkan(v) => v,
+        _ => eyre::bail!("Wrong Graphics Api"),
+    };
     let (session, frame_wait, frame_stream) = unsafe {
         xr_instance.create_session::<xr::Vulkan>(
-            xr_system_id,
+            xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY)?,
             &xr::vulkan::SessionCreateInfo {
-                instance: vk_instance_ptr,
-                physical_device: vk_physical_device_ptr,
-                device: vk_device_ptr,
-                queue_family_index,
+                instance: setup_info.vk_instance_ptr,
+                physical_device: setup_info.physical_device_ptr,
+                device: setup_info.device_ptr,
+                queue_family_index: setup_info.queue_family_index,
                 queue_index: 0,
             },
         )
     }?;
 
-    let views = xr_instance.enumerate_view_configuration_views(xr_system_id, VIEW_TYPE)?;
-
+    let views =
+        xr_instance.enumerate_view_configuration_views(setup_info.xr_system_id, VIEW_TYPE)?;
     let surface = window.map(|wrapper| unsafe {
         // SAFETY: Plugins should be set up on the main thread.
         let handle = wrapper.get_handle();
         wgpu_instance
-            .create_surface(&handle)
+            .create_surface(handle)
             .expect("Failed to create wgpu surface")
     });
     let swapchain_format = surface
         .as_ref()
-        .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0])
+        .map(|surface| surface.get_capabilities(wgpu_adapter).formats[0])
         .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb);
 
     // TODO: Log swapchain format
@@ -356,11 +389,13 @@ pub fn initialize_xr_graphics(
             mip_count: 1,
         })
         .unwrap();
+
     let images = handle.enumerate_images().unwrap();
 
     let buffers = images
         .into_iter()
         .map(|color_image| {
+            info!("image map swapchain");
             let color_image = vk::Image::from_raw(color_image);
             let wgpu_hal_texture = unsafe {
                 <V as Api>::Device::texture_from_raw(
@@ -409,18 +444,12 @@ pub fn initialize_xr_graphics(
         .collect();
 
     Ok((
-        wgpu_device.into(),
-        RenderQueue(Arc::new(wgpu_queue)),
-        RenderAdapterInfo(wgpu_adapter.get_info()),
-        RenderAdapter(Arc::new(wgpu_adapter)),
-        wgpu_instance,
-        xr_instance.clone().into(),
-        session.clone().into_any_graphics().into(),
-        blend_mode.into(),
+        XrSession::Vulkan(session.clone()),
         resolution.into(),
         swapchain_format.into(),
+        // TODO: this shouldn't be in here
         AtomicBool::new(false).into(),
-        Mutex::new(frame_wait).into(),
+        frame_wait.into(),
         Swapchain::Vulkan(SwapchainInner {
             stream: Mutex::new(frame_stream),
             handle: Mutex::new(handle),
@@ -428,13 +457,14 @@ pub fn initialize_xr_graphics(
             image_index: Mutex::new(0),
         })
         .into(),
-        XrInput::new(xr_instance, session.into_any_graphics())?,
-        Mutex::default().into(),
-        Mutex::new(xr::FrameState {
+        XrInput::new(xr_instance, &session.into_any_graphics())?,
+        Vec::default().into(),
+        // TODO: Feels wrong to return a FrameState here, we probably should just wait for the next frame
+        xr::FrameState {
             predicted_display_time: xr::Time::from_nanos(1),
             predicted_display_period: xr::Duration::from_nanos(1),
             should_render: true,
-        })
+        }
         .into(),
     ))
 }

src/input.rs

@@ -1,10 +1,10 @@
 use std::sync::Arc;
 
-use bevy::prelude::*;
+use bevy::{prelude::*, render::extract_resource::ExtractResource};
 use openxr as xr;
 use xr::{FrameState, FrameWaiter, ViewConfigurationType};
 
-#[derive(Clone, Resource)]
+#[derive(Clone, Resource, ExtractResource)]
 pub struct XrInput {
     //pub action_set: xr::ActionSet,
     //pub hand_pose: xr::Action<xr::Posef>,
@@ -16,8 +16,8 @@ pub struct XrInput {
 
 impl XrInput {
     pub fn new(
-        instance: xr::Instance,
-        session: xr::Session<xr::AnyGraphics>,
+        instance: &xr::Instance,
+        session: &xr::Session<xr::AnyGraphics>,
         // frame_state: &FrameState,
     ) -> xr::Result<Self> {
         // let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap();

src/lib.rs

@@ -1,17 +1,19 @@
 pub mod graphics;
 pub mod input;
 pub mod passthrough;
+pub mod prelude;
 pub mod resource_macros;
 pub mod resources;
 pub mod xr_init;
 pub mod xr_input;
 
-use std::sync::{Arc, Mutex};
+use std::sync::atomic::AtomicBool;
 
-use crate::xr_init::RenderRestartPlugin;
-use crate::xr_input::hands::hand_tracking::DisableHandTracking;
+use crate::xr_init::{StartXrSession, XrInitPlugin};
 use crate::xr_input::oculus_touch::ActionSets;
+use crate::xr_input::trackers::verify_quat;
 use bevy::app::{AppExit, PluginGroupBuilder};
+use bevy::core::TaskPoolThreadAssignmentPolicy;
 use bevy::ecs::system::SystemState;
 use bevy::prelude::*;
 use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews};
@@ -24,14 +26,19 @@ use graphics::extensions::XrExtensions;
 use graphics::{XrAppInfo, XrPreferdBlendMode};
 use input::XrInput;
 use openxr as xr;
-use passthrough::{start_passthrough, supports_passthrough, Passthrough, PassthroughLayer};
+use passthrough::{PassthroughPlugin, XrPassthroughLayer, XrPassthroughState};
 use resources::*;
-use xr::FormFactor;
-use xr_init::{xr_only, XrEnableStatus, XrRenderData};
-use xr_input::controllers::XrControllerType;
+use xr_init::{
+    xr_after_wait_only, xr_only, xr_render_only, CleanupRenderWorld, CleanupXrData,
+    ExitAppOnSessionExit, SetupXrData, StartSessionOnStartup, XrCleanup, XrEarlyInitPlugin,
+    XrHasWaited, XrPostCleanup, XrShouldRender, XrStatus,
+};
+use xr_input::actions::XrActionsPlugin;
 use xr_input::hands::emulated::HandEmulationPlugin;
-use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin};
-use xr_input::OpenXrInput;
+use xr_input::hands::hand_tracking::HandTrackingPlugin;
+use xr_input::hands::HandPlugin;
+use xr_input::xr_camera::XrCameraPlugin;
+use xr_input::XrInputPlugin;
 
 const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
 
@@ -46,225 +53,170 @@ pub struct OpenXrPlugin {
     app_info: XrAppInfo,
 }
 
-#[derive(Resource)]
-pub struct FutureXrResources(
-    pub  Arc<
-        Mutex<
-            Option<(
-                XrInstance,
-                XrSession,
-                XrEnvironmentBlendMode,
-                XrResolution,
-                XrFormat,
-                XrSessionRunning,
-                XrFrameWaiter,
-                XrSwapchain,
-                XrInput,
-                XrViews,
-                XrFrameState,
-                bool,
-                XrPassthrough,
-                XrPassthroughLayer,
-            )>,
-        >,
-    >,
-);
-fn mr_test(mut commands: Commands, passthrough_layer: Option<Res<XrPassthroughLayer>>) {
-    commands.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0)));
-}
-
 impl Plugin for OpenXrPlugin {
     fn build(&self, app: &mut App) {
-        let mut system_state: SystemState<Query<&RawHandleWrapper, With<PrimaryWindow>>> =
-            SystemState::new(&mut app.world);
-        let primary_window = system_state.get(&app.world).get_single().ok().cloned();
-
+        app.insert_resource(XrSessionRunning::new(AtomicBool::new(false)));
+        app.insert_resource(ExitAppOnSessionExit::default());
         #[cfg(not(target_arch = "wasm32"))]
-        match graphics::initialize_xr_graphics(
-            primary_window.clone(),
+        match graphics::initialize_xr_instance(
+            SystemState::<Query<&RawHandleWrapper, With<PrimaryWindow>>>::new(&mut app.world)
+                .get(&app.world)
+                .get_single()
+                .ok()
+                .cloned(),
             self.reqeusted_extensions.clone(),
             self.prefered_blend_mode,
             self.app_info.clone(),
         ) {
             Ok((
+                xr_instance,
+                oxr_session_setup_info,
+                blend_mode,
                 device,
                 queue,
                 adapter_info,
                 render_adapter,
                 instance,
-                xr_instance,
-                session,
-                blend_mode,
-                resolution,
-                format,
-                session_running,
-                frame_waiter,
-                swapchain,
-                input,
-                views,
-                frame_state,
             )) => {
-                // std::thread::sleep(Duration::from_secs(5));
                 debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
                 debug!("Configured wgpu adapter Features: {:#?}", device.features());
+                warn!("Starting with OpenXR Instance");
                 app.insert_resource(xr_instance.clone());
-                app.insert_resource(session.clone());
-                app.insert_resource(blend_mode.clone());
-                app.insert_resource(resolution.clone());
-                app.insert_resource(format.clone());
-                app.insert_resource(session_running.clone());
-                app.insert_resource(frame_waiter.clone());
-                app.insert_resource(swapchain.clone());
-                app.insert_resource(input.clone());
-                app.insert_resource(views.clone());
-                app.insert_resource(frame_state.clone());
-
-                // Check if the fb_passthrough extension is available
-                let fb_passthrough_available = xr_instance.exts().fb_passthrough.is_some();
-                bevy::log::info!(
-                    "From OpenXrPlugin: fb_passthrough_available: {}",
-                    fb_passthrough_available
-                );
-                // Get the system for the head-mounted display
-                let hmd_system = xr_instance
-                    .system(FormFactor::HEAD_MOUNTED_DISPLAY)
-                    .unwrap();
-                bevy::log::info!("From OpenXrPlugin: hmd_system: {:?}", hmd_system);
-
-                // Check if the system supports passthrough
-                let passthrough_supported =
-                    supports_passthrough(&xr_instance, hmd_system).is_ok_and(|v| v);
-                bevy::log::info!(
-                    "From OpenXrPlugin: passthrough_supported: {}",
-                    passthrough_supported
-                );
-
-                // The passthrough variable will be true only if both fb_passthrough is available and the system supports passthrough
-                let passthrough = fb_passthrough_available && passthrough_supported;
-                bevy::log::info!("From OpenXrPlugin: passthrough: {}", passthrough);
-
-                let mut p: Option<XrPassthrough> = None;
-                let mut pl: Option<XrPassthroughLayer> = None;
-                if passthrough {
-                    if let Ok((p, pl)) = start_passthrough(&xr_instance, &session) {
-                        let xr_data = XrRenderData {
-                            xr_instance,
-                            xr_session: session,
-                            xr_blend_mode: blend_mode,
-                            xr_resolution: resolution,
-                            xr_format: format,
-                            xr_session_running: session_running,
-                            xr_frame_waiter: frame_waiter,
-                            xr_swapchain: swapchain,
-                            xr_input: input,
-                            xr_views: views,
-                            xr_frame_state: frame_state,
-                            xr_passthrough_active: true,
-                            xr_passthrough: XrPassthrough::new(Mutex::new(p)),
-                            xr_passthrough_layer: XrPassthroughLayer::new(Mutex::new(pl)),
-                        };
-                        bevy::log::info!("Passthrough is supported!");
-                        app.insert_resource(xr_data);
-                        app.insert_resource(ClearColor(Color::rgba(0.0, 0.0, 0.0, 0.0)));
-                    }
-
-                    if !app.world.contains_resource::<ClearColor>() {
-                        info!("ClearColor!");
-                    }
-                }
+                app.insert_resource(blend_mode);
                 app.insert_resource(ActionSets(vec![]));
+                app.insert_resource(xr_instance);
+                app.insert_resource(blend_mode);
+                app.insert_non_send_resource(oxr_session_setup_info);
+                let render_instance = RenderInstance(instance.into());
+                app.insert_resource(render_instance.clone());
                 app.add_plugins(RenderPlugin {
                     render_creation: RenderCreation::Manual(
                         device,
                         queue,
                         adapter_info,
                         render_adapter,
-                        RenderInstance(Arc::new(instance)),
+                        render_instance,
                     ),
+                    // Expose this? if yes we also have to set this in the non xr case
+                    synchronous_pipeline_compilation: true,
                 });
-                app.insert_resource(XrEnableStatus::Enabled);
+                app.insert_resource(XrStatus::Disabled);
+                // app.world.send_event(StartXrSession);
             }
             Err(err) => {
-                warn!("OpenXR Failed to initialize: {}", err);
+                warn!("OpenXR Instance Failed to initialize: {}", err);
                 app.add_plugins(RenderPlugin::default());
-                app.insert_resource(XrEnableStatus::Disabled);
+                app.insert_resource(XrStatus::NoInstance);
             }
         }
-        // app.add_systems(PreUpdate, mr_test);
         #[cfg(target_arch = "wasm32")]
         {
             app.add_plugins(RenderPlugin::default());
-            app.insert_resource(XrEnableStatus::Disabled);
+            app.insert_resource(XrStatus::Disabled);
         }
-    }
-
-    fn ready(&self, app: &App) -> bool {
-        app.world
-            .get_resource::<XrEnableStatus>()
-            .map(|frr| *frr != XrEnableStatus::Waiting)
-            .unwrap_or(true)
-    }
-
-    fn finish(&self, app: &mut App) {
-        // TODO: Split this up into the indevidual resources
-        if let Some(data) = app.world.get_resource::<XrRenderData>().cloned() {
-            let hands = data.xr_instance.exts().ext_hand_tracking.is_some()
-                && data
-                    .xr_instance
-                    .supports_hand_tracking(
-                        data.xr_instance
-                            .system(FormFactor::HEAD_MOUNTED_DISPLAY)
-                            .unwrap(),
+        app.add_systems(XrPostCleanup, clean_resources);
+        app.add_systems(XrPostCleanup, || info!("Main World Post Cleanup!"));
+        app.add_systems(
+            PreUpdate,
+            xr_poll_events.run_if(|status: Res<XrStatus>| *status != XrStatus::NoInstance),
+        );
+        app.add_systems(
+            PreUpdate,
+            (
+                xr_reset_per_frame_resources,
+                xr_wait_frame.run_if(xr_only()),
+                locate_views.run_if(xr_only()),
+                apply_deferred,
             )
-                    .is_ok_and(|v| v);
-            if hands {
-                app.insert_resource(HandTrackingData::new(&data.xr_session).unwrap());
-            } else {
-                app.insert_resource(DisableHandTracking::Both);
-            }
-
-            let (left, right) = data.xr_swapchain.get_render_views();
-            let left = ManualTextureView {
-                texture_view: left.into(),
-                size: *data.xr_resolution,
-                format: *data.xr_format,
-            };
-            let right = ManualTextureView {
-                texture_view: right.into(),
-                size: *data.xr_resolution,
-                format: *data.xr_format,
-            };
-            app.add_systems(PreUpdate, xr_begin_frame.run_if(xr_only()));
-            let mut manual_texture_views = app.world.resource_mut::<ManualTextureViews>();
-            manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left);
-            manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right);
-            drop(manual_texture_views);
+                .chain()
+                .after(xr_poll_events),
+        );
         let render_app = app.sub_app_mut(RenderApp);
-
-            render_app.insert_resource(data.xr_instance.clone());
-            render_app.insert_resource(data.xr_session.clone());
-            render_app.insert_resource(data.xr_blend_mode.clone());
-            render_app.insert_resource(data.xr_resolution.clone());
-            render_app.insert_resource(data.xr_format.clone());
-            render_app.insert_resource(data.xr_session_running.clone());
-            render_app.insert_resource(data.xr_frame_waiter.clone());
-            render_app.insert_resource(data.xr_swapchain.clone());
-            render_app.insert_resource(data.xr_input.clone());
-            render_app.insert_resource(data.xr_views.clone());
-            render_app.insert_resource(data.xr_frame_state.clone());
-            render_app.insert_resource(data.xr_passthrough.clone());
-            render_app.insert_resource(data.xr_passthrough_layer.clone());
-            render_app.insert_resource(XrEnableStatus::Enabled);
         render_app.add_systems(
             Render,
-                (
-                    post_frame
+            xr_pre_frame
                 .run_if(xr_only())
+                .run_if(xr_after_wait_only())
+                .run_if(xr_render_only())
                 .before(render_system)
                 .after(RenderSet::ExtractCommands),
-                    end_frame.run_if(xr_only()).after(render_system),
-                ),
         );
+        render_app.add_systems(
+            Render,
+            xr_end_frame
+                .run_if(xr_only())
+                .run_if(xr_after_wait_only())
+                .run_if(xr_render_only())
+                .in_set(RenderSet::Cleanup),
+        );
+        render_app.add_systems(
+            Render,
+            xr_skip_frame
+                .run_if(xr_only())
+                .run_if(xr_after_wait_only())
+                .run_if(not(xr_render_only()))
+                .in_set(RenderSet::Cleanup),
+        );
+        render_app.add_systems(
+            Render,
+            clean_resources_render
+                .run_if(resource_exists::<CleanupRenderWorld>)
+                .after(RenderSet::ExtractCommands),
+        );
+    }
+}
+
+fn clean_resources_render(mut cmds: &mut World) {
+    // let session = cmds.remove_resource::<XrSession>().unwrap();
+    cmds.remove_resource::<XrSession>();
+    cmds.remove_resource::<XrResolution>();
+    cmds.remove_resource::<XrFormat>();
+    // cmds.remove_resource::<XrSessionRunning>();
+    cmds.remove_resource::<XrFrameWaiter>();
+    cmds.remove_resource::<XrSwapchain>();
+    cmds.remove_resource::<XrInput>();
+    cmds.remove_resource::<XrViews>();
+    cmds.remove_resource::<XrFrameState>();
+    cmds.remove_resource::<CleanupRenderWorld>();
+    // unsafe {
+    //     (session.instance().fp().destroy_session)(session.as_raw());
+    // }
+    warn!("Cleanup Resources Render");
+}
+fn clean_resources(mut cmds: &mut World) {
+    cmds.remove_resource::<XrSession>();
+    cmds.remove_resource::<XrResolution>();
+    cmds.remove_resource::<XrFormat>();
+    // cmds.remove_resource::<XrSessionRunning>();
+    cmds.remove_resource::<XrFrameWaiter>();
+    cmds.remove_resource::<XrSwapchain>();
+    cmds.remove_resource::<XrInput>();
+    cmds.remove_resource::<XrViews>();
+    cmds.remove_resource::<XrFrameState>();
+    // cmds.remove_resource::<CleanupRenderWorld>();
+    // unsafe {
+    //     (session.instance().fp().destroy_session)(session.as_raw());
+    // }
+    warn!("Cleanup Resources");
+}
+
+fn xr_skip_frame(
+    xr_swapchain: Res<XrSwapchain>,
+    xr_frame_state: Res<XrFrameState>,
+    environment_blend_mode: Res<XrEnvironmentBlendMode>,
+) {
+    let swapchain: &Swapchain = &xr_swapchain;
+    match swapchain {
+        Swapchain::Vulkan(swap) => {
+            swap.stream
+                .lock()
+                .unwrap()
+                .end(
+                    xr_frame_state.predicted_display_time,
+                    **environment_blend_mode,
+                    &[],
+                )
+                .unwrap();
         }
     }
 }
@@ -280,17 +232,35 @@ impl PluginGroup for DefaultXrPlugins {
     fn build(self) -> PluginGroupBuilder {
         DefaultPlugins
             .build()
+            .set(TaskPoolPlugin {
+                task_pool_options: TaskPoolOptions {
+                    compute: TaskPoolThreadAssignmentPolicy {
+                        min_threads: 2,
+                        max_threads: std::usize::MAX, // unlimited max threads
+                        percent: 1.0,                 // this value is irrelevant in this case
+                    },
+                    // keep the defaults for everything else
+                    ..default()
+                },
+            })
+            // .disable::<PipelinedRenderingPlugin>()
             .disable::<RenderPlugin>()
-            .disable::<PipelinedRenderingPlugin>()
             .add_before::<RenderPlugin, _>(OpenXrPlugin {
                 prefered_blend_mode: self.prefered_blend_mode,
                 reqeusted_extensions: self.reqeusted_extensions,
                 app_info: self.app_info.clone(),
             })
-            .add_after::<OpenXrPlugin, _>(OpenXrInput::new(XrControllerType::OculusTouch))
-            .add_before::<OpenXrPlugin, _>(RenderRestartPlugin)
-            .add(HandEmulationPlugin)
+            .add_after::<OpenXrPlugin, _>(XrInitPlugin)
+            .add(XrInputPlugin)
+            .add(XrActionsPlugin)
+            .add(XrCameraPlugin)
+            .add_before::<OpenXrPlugin, _>(XrEarlyInitPlugin)
+            .add(HandPlugin)
             .add(HandTrackingPlugin)
+            .add(HandEmulationPlugin)
+            .add(PassthroughPlugin)
+            .add(XrResourcePlugin)
+            .add(StartSessionOnStartup)
             .set(WindowPlugin {
                 #[cfg(not(target_os = "android"))]
                 primary_window: Some(Window {
@@ -310,18 +280,25 @@ impl PluginGroup for DefaultXrPlugins {
     }
 }
 
-pub fn xr_begin_frame(
-    instance: Res<XrInstance>,
-    session: Res<XrSession>,
-    session_running: Res<XrSessionRunning>,
-    frame_state: Res<XrFrameState>,
-    frame_waiter: Res<XrFrameWaiter>,
-    swapchain: Res<XrSwapchain>,
-    views: Res<XrViews>,
-    input: Res<XrInput>,
-    mut app_exit: EventWriter<AppExit>,
+fn xr_reset_per_frame_resources(
+    mut should: ResMut<XrShouldRender>,
+    mut waited: ResMut<XrHasWaited>,
 ) {
-    {
+    **should = false;
+    **waited = false;
+}
+
+fn xr_poll_events(
+    instance: Option<Res<XrInstance>>,
+    session: Option<Res<XrSession>>,
+    session_running: Res<XrSessionRunning>,
+    exit_type: Res<ExitAppOnSessionExit>,
+    mut app_exit: EventWriter<AppExit>,
+    mut start_session: EventWriter<StartXrSession>,
+    mut setup_xr: EventWriter<SetupXrData>,
+    mut cleanup_xr: EventWriter<CleanupXrData>,
+) {
+    if let (Some(instance), Some(session)) = (instance, session) {
         let _span = info_span!("xr_poll_events");
         while let Some(event) = instance.poll_event(&mut Default::default()).unwrap() {
             use xr::Event::*;
@@ -332,22 +309,38 @@ pub fn xr_begin_frame(
                     info!("entered XR state {:?}", e.state());
                     match e.state() {
                         xr::SessionState::READY => {
+                            info!("Calling Session begin :3");
                             session.begin(VIEW_TYPE).unwrap();
+                            setup_xr.send_default();
                             session_running.store(true, std::sync::atomic::Ordering::Relaxed);
                         }
                         xr::SessionState::STOPPING => {
                             session.end().unwrap();
                             session_running.store(false, std::sync::atomic::Ordering::Relaxed);
-                            app_exit.send(AppExit);
+                            cleanup_xr.send_default();
                         }
-                        xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
-                            app_exit.send(AppExit);
-                            return;
+                        xr::SessionState::EXITING => {
+                            if *exit_type == ExitAppOnSessionExit::Always
+                                || *exit_type == ExitAppOnSessionExit::OnlyOnExit
+                            {
+                                app_exit.send_default();
                             }
+                        }
+                        xr::SessionState::LOSS_PENDING => {
+                            if *exit_type == ExitAppOnSessionExit::Always {
+                                app_exit.send_default();
+                            }
+                            if *exit_type == ExitAppOnSessionExit::OnlyOnExit {
+                                start_session.send_default();
+                            }
+                        }
+
                         _ => {}
                     }
                 }
-                InstanceLossPending(_) => return,
+                InstanceLossPending(_) => {
+                    app_exit.send_default();
+                }
                 EventsLost(e) => {
                     warn!("lost {} XR events", e.lost_event_count());
                 }
@@ -355,34 +348,43 @@ pub fn xr_begin_frame(
             }
         }
     }
+}
+
+pub fn xr_wait_frame(
+    world: &mut World,
+    // mut frame_state: ResMut<XrFrameState>,
+    // mut frame_waiter: ResMut<XrFrameWaiter>,
+    // mut should_render: ResMut<XrShouldRender>,
+    // mut waited: ResMut<XrHasWaited>,
+) {
+    let mut frame_waiter = world.get_resource_mut::<XrFrameWaiter>().unwrap();
     {
         let _span = info_span!("xr_wait_frame").entered();
-        *frame_state.lock().unwrap() = match frame_waiter.lock().unwrap().wait() {
-            Ok(a) => a,
+
+        *world.get_resource_mut::<XrFrameState>().unwrap() = match frame_waiter.wait() {
+            Ok(a) => a.into(),
             Err(e) => {
                 warn!("error: {}", e);
                 return;
             }
         };
+        let should_render = world.get_resource::<XrFrameState>().unwrap().should_render;
+        let mut frame_state = world.resource_mut::<XrFrameState>();
+        frame_state.predicted_display_time = xr::Time::from_nanos(
+            frame_state.predicted_display_time.as_nanos()
+                + frame_state.predicted_display_period.as_nanos(),
+        );
+        **world.get_resource_mut::<XrShouldRender>().unwrap() = should_render;
+        **world.get_resource_mut::<XrHasWaited>().unwrap() = true;
     }
-    {
-        let _span = info_span!("xr_begin_frame").entered();
-        swapchain.begin().unwrap()
-    }
-    {
-        let _span = info_span!("xr_locate_views").entered();
-        *views.lock().unwrap() = session
-            .locate_views(
-                VIEW_TYPE,
-                frame_state.lock().unwrap().predicted_display_time,
-                &input.stage,
-            )
+    world
+        .get_resource::<XrSwapchain>()
         .unwrap()
-            .1;
-    }
+        .begin()
+        .unwrap();
 }
 
-pub fn post_frame(
+pub fn xr_pre_frame(
     resolution: Res<XrResolution>,
     format: Res<XrFormat>,
     swapchain: Res<XrSwapchain>,
@@ -414,7 +416,8 @@ pub fn post_frame(
     }
 }
 
-pub fn end_frame(
+#[allow(clippy::too_many_arguments)]
+pub fn xr_end_frame(
     xr_frame_state: Res<XrFrameState>,
     views: Res<XrViews>,
     input: Res<XrInput>,
@@ -422,6 +425,7 @@ pub fn end_frame(
     resolution: Res<XrResolution>,
     environment_blend_mode: Res<XrEnvironmentBlendMode>,
     passthrough_layer: Option<Res<XrPassthroughLayer>>,
+    passthrough_state: Option<Res<XrPassthroughState>>,
 ) {
     #[cfg(target_os = "android")]
     {
@@ -436,18 +440,17 @@ pub fn end_frame(
     }
     {
         let _span = info_span!("xr_end_frame").entered();
-        // bevy::log::info!(
-        //     "passthrough_layer.is_some(): {:?}",
-        //     passthrough_layer.is_some()
-        // );
-
+        let pass_layer = match passthrough_state.as_deref() {
+            Some(XrPassthroughState::Running) => passthrough_layer.as_deref(),
+            _ => None,
+        };
         let result = swapchain.end(
-            xr_frame_state.lock().unwrap().predicted_display_time,
-            &views.lock().unwrap(),
+            xr_frame_state.predicted_display_time,
+            &views,
             &input.stage,
             **resolution,
             **environment_blend_mode,
-            passthrough_layer.map(|p| PassthroughLayer(*p.lock().unwrap())),
+            pass_layer,
         );
         match result {
             Ok(_) => {}
@@ -457,22 +460,37 @@ pub fn end_frame(
 }
 
 pub fn locate_views(
-    views: Res<XrViews>,
+    mut views: ResMut<XrViews>,
     input: Res<XrInput>,
     session: Res<XrSession>,
     xr_frame_state: Res<XrFrameState>,
 ) {
     let _span = info_span!("xr_locate_views").entered();
-    *views.lock().unwrap() = match session.locate_views(
+    **views = match session.locate_views(
         VIEW_TYPE,
-        xr_frame_state.lock().unwrap().predicted_display_time,
+        xr_frame_state.predicted_display_time,
         &input.stage,
     ) {
-        Ok(this) => this,
+        Ok(this) => this
+            .1
+            .into_iter()
+            .map(|mut view| {
+                use crate::prelude::*;
+                let quat = view.pose.orientation.to_quat();
+                let fixed_quat = verify_quat(quat);
+                let oxr_quat = xr::Quaternionf {
+                    x: fixed_quat.x,
+                    y: fixed_quat.y,
+                    z: fixed_quat.z,
+                    w: fixed_quat.w,
+                };
+                view.pose.orientation = oxr_quat;
+                view
+            })
+            .collect(),
         Err(err) => {
             warn!("error: {}", err);
             return;
         }
     }
-    .1;
 }

src/passthrough.rs

@@ -1,20 +1,135 @@
-use bevy::prelude::*;
+use bevy::render::extract_resource::ExtractResource;
+use bevy::{prelude::*, render::extract_resource::ExtractResourcePlugin};
 use std::{marker::PhantomData, mem, ptr};
 
-use crate::xr_init::XrRenderData;
+use crate::resources::XrSession;
+use crate::{
+    resources::XrInstance,
+    xr_arc_resource_wrapper,
+    xr_init::{XrCleanup, XrSetup},
+};
 use openxr as xr;
 use xr::{
-    sys::{
-        PassthroughCreateInfoFB, PassthroughFB, PassthroughLayerFB, Space,
-        SystemPassthroughProperties2FB,
-    },
-    CompositionLayerBase, CompositionLayerFlags, Graphics, PassthroughCapabilityFlagsFB,
+    sys::{Space, SystemPassthroughProperties2FB},
+    CompositionLayerBase, CompositionLayerFlags, FormFactor, Graphics,
+    PassthroughCapabilityFlagsFB,
 };
 
-use crate::resources::XrInstance;
-use crate::resources::XrSession;
-pub struct PassthroughLayer(pub xr::sys::PassthroughLayerFB);
-pub struct Passthrough(pub xr::sys::PassthroughFB);
+#[derive(
+    Clone, Copy, Default, Debug, Resource, PartialEq, PartialOrd, Ord, Eq, Reflect, ExtractResource,
+)]
+pub enum XrPassthroughState {
+    #[default]
+    Unsupported,
+    Running,
+    Paused,
+}
+
+xr_arc_resource_wrapper!(XrPassthrough, xr::Passthrough);
+xr_arc_resource_wrapper!(XrPassthroughLayer, xr::PassthroughLayer);
+
+pub struct PassthroughPlugin;
+
+impl Plugin for PassthroughPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_event::<ResumePassthrough>();
+        app.add_event::<PausePassthrough>();
+        app.add_plugins(ExtractResourcePlugin::<XrPassthroughLayer>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrPassthroughState>::default());
+        app.register_type::<XrPassthroughState>();
+        app.add_systems(Startup, check_passthrough_support);
+        app.add_systems(
+            XrSetup,
+            setup_passthrough
+                .run_if(|state: Res<XrPassthroughState>| *state != XrPassthroughState::Unsupported),
+        );
+        app.add_systems(XrCleanup, cleanup_passthrough);
+        app.add_systems(
+            Update,
+            resume_passthrough.run_if(
+                resource_exists_and_equals(XrPassthroughState::Paused)
+                    .and_then(on_event::<ResumePassthrough>()),
+            ),
+        );
+        app.add_systems(
+            Update,
+            pause_passthrough.run_if(
+                resource_exists_and_equals(XrPassthroughState::Running)
+                    .and_then(on_event::<PausePassthrough>()),
+            ),
+        );
+    }
+}
+
+fn check_passthrough_support(mut cmds: Commands, instance: Option<Res<XrInstance>>) {
+    match instance {
+        None => cmds.insert_resource(XrPassthroughState::Unsupported),
+        Some(instance) => {
+            let supported = instance.exts().fb_passthrough.is_some()
+                && supports_passthrough(
+                    &instance,
+                    instance.system(FormFactor::HEAD_MOUNTED_DISPLAY).unwrap(),
+                )
+                .is_ok_and(|v| v);
+            match supported {
+                false => cmds.insert_resource(XrPassthroughState::Unsupported),
+                true => cmds.insert_resource(XrPassthroughState::Paused),
+            }
+        }
+    }
+}
+
+fn resume_passthrough(
+    layer: Res<XrPassthroughLayer>,
+    mut state: ResMut<XrPassthroughState>,
+    mut clear_color: ResMut<ClearColor>,
+) {
+    if let Err(e) = layer.resume() {
+        warn!("Unable to resume Passthrough: {}", e);
+        return;
+    }
+    clear_color.set_a(0.0);
+    clear_color.set_r(0.0);
+    clear_color.set_g(0.0);
+    clear_color.set_b(0.0);
+    *state = XrPassthroughState::Running;
+}
+fn pause_passthrough(
+    layer: Res<XrPassthroughLayer>,
+    mut state: ResMut<XrPassthroughState>,
+    mut clear_color: ResMut<ClearColor>,
+) {
+    if let Err(e) = layer.pause() {
+        warn!("Unable to resume Passthrough: {}", e);
+        return;
+    }
+    clear_color.set_a(1.0);
+    *state = XrPassthroughState::Paused;
+}
+
+fn cleanup_passthrough(mut cmds: Commands) {
+    cmds.remove_resource::<XrPassthrough>();
+    cmds.remove_resource::<XrPassthroughLayer>();
+}
+
+fn setup_passthrough(mut cmds: Commands, session: Res<XrSession>) {
+    match create_passthrough(&session) {
+        Ok((passthrough, layer)) => {
+            cmds.insert_resource(XrPassthrough::from(passthrough));
+            cmds.insert_resource(XrPassthroughLayer::from(layer));
+        }
+        Err(e) => {
+            warn!("Unable to create passthrough: {}", e);
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default, Reflect, Event)]
+pub struct ResumePassthrough;
+
+#[derive(Clone, Copy, Debug, Default, Reflect, Event)]
+pub struct PausePassthrough;
+
 fn cvt(x: xr::sys::Result) -> xr::Result<xr::sys::Result> {
     if x.into_raw() >= 0 {
         Ok(x)
@@ -38,14 +153,14 @@ impl<'a, G: Graphics> std::ops::Deref for CompositionLayerPassthrough<'a, G> {
 }
 
 impl<'a, G: xr::Graphics> CompositionLayerPassthrough<'a, G> {
-    pub(crate) fn from_xr_passthrough_layer(layer: &PassthroughLayer) -> Self {
+    pub(crate) fn from_xr_passthrough_layer(layer: &XrPassthroughLayer) -> Self {
         Self {
             inner: xr::sys::CompositionLayerPassthroughFB {
                 ty: xr::sys::CompositionLayerPassthroughFB::TYPE,
                 next: ptr::null(),
                 flags: CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA,
                 space: Space::NULL,
-                layer_handle: layer.0,
+                layer_handle: *layer.inner(),
             },
             _marker: PhantomData,
         }
@@ -77,103 +192,32 @@ pub fn supports_passthrough(instance: &XrInstance, system: xr::SystemId) -> xr::
 }
 
 #[inline]
-pub fn start_passthrough(
-    instance: &XrInstance,
+pub fn create_passthrough(
     xr_session: &XrSession,
-) -> xr::Result<(xr::sys::PassthroughFB, xr::sys::PassthroughLayerFB)> {
-    unsafe {
-        // Create feature
-        let mut passthrough_feature = xr::sys::PassthroughFB::NULL;
-        let mut passthrough_create_info = xr::sys::PassthroughCreateInfoFB {
-            ty: xr::sys::StructureType::PASSTHROUGH_CREATE_INFO_FB, // XR_TYPE_PASSTHROUGH_CREATE_INFO_FB
-            next: ptr::null(),
-            flags: xr::sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION,
-        };
-        // bevy::log::info!("xr_session.as_raw(): {:?}", xr_session.as_raw());
-        // bevy::log::info!("&passthrough_create_info: {:?}", &passthrough_create_info);
-        // bevy::log::info!("&mut passthrough_feature: {:?}", &mut passthrough_feature);
-        // bevy::log::info!(
-        //     "instance.exts().fb_passthrough.unwrap(): {:?}",
-        //     instance.exts().fb_passthrough.is_some()
-        // );
-        cvt(
-            (instance.exts().fb_passthrough.unwrap().create_passthrough)(
-                xr_session.as_raw(),
-                &passthrough_create_info as *const _,
-                &mut passthrough_feature as *mut _,
+) -> xr::Result<(xr::Passthrough, xr::PassthroughLayer)> {
+    let passthrough = match xr_session {
+        XrSession::Vulkan(session) => {
+            session.create_passthrough(xr::PassthroughFlagsFB::IS_RUNNING_AT_CREATION)
+        }
+    }?;
+    let passthrough_layer = match xr_session {
+        XrSession::Vulkan(session) => session.create_passthrough_layer(
+            &passthrough,
+            xr::PassthroughFlagsFB::IS_RUNNING_AT_CREATION,
+            xr::PassthroughLayerPurposeFB::RECONSTRUCTION,
         ),
-        )?;
-        // bevy::log::info!("Created passthrough feature");
-        // Create layer
-        let mut passthrough_layer = xr::sys::PassthroughLayerFB::NULL;
-        let mut layer_create_info: xr::sys::PassthroughLayerCreateInfoFB =
-            xr::sys::PassthroughLayerCreateInfoFB {
-                ty: xr::sys::StructureType::PASSTHROUGH_LAYER_CREATE_INFO_FB, // XR_TYPE_PASSTHROUGH_LAYER_CREATE_INFO_FB
-                next: ptr::null(),
-                passthrough: passthrough_feature, // XR_PASSTHROUGH_HANDLE
-                flags: xr::sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION, // XR_PASSTHROUGH_IS_RUNNING_AT_CREATION_BIT_FB
-                purpose: xr::sys::PassthroughLayerPurposeFB::RECONSTRUCTION, // XR_PASSTHROUGH_LAYER_PURPOSE_RECONSTRUCTION_FB
-            };
-        cvt((instance
-            .exts()
-            .fb_passthrough
-            .unwrap()
-            .create_passthrough_layer)(
-            xr_session.as_raw(),
-            &layer_create_info as *const _,
-            &mut passthrough_layer as *mut _,
-        ))?;
-        // bevy::log::info!("Created passthrough layer");
-        // // Start layer
-
-        // bevy::log::info!("passthrough_feature: {:?}", passthrough_feature);
-        // // cvt((instance.exts().fb_passthrough.unwrap().passthrough_start)(
-        // //     passthrough_feature,
-        // // ))?;
-        // bevy::log::info!("Started passthrough layer");
-        // bevy::log::info!("Passed everything in  start");
-        Ok((passthrough_feature, passthrough_layer))
-    }
+    }?;
+    Ok((passthrough, passthrough_layer))
 }
 
-#[inline]
-pub fn passthrough_layer_resume(mut xr_data_resource: ResMut<XrRenderData>) -> xr::Result<()> {
-    unsafe {
-        let passthrough_layer = &xr_data_resource.xr_passthrough_layer;
-        {
-            let passthrough_layer_locked = passthrough_layer.lock().unwrap();
-            cvt((xr_data_resource
-                .xr_instance
-                .exts()
-                .fb_passthrough
-                .unwrap()
-                .passthrough_layer_resume)(
-                *passthrough_layer_locked
-            ))?;
-        }
-        xr_data_resource.xr_passthrough_active = true;
-        bevy::log::info!("Resumed passthrough layer");
-        Ok(())
-    }
-}
+/// Enable Passthrough on xr startup
+/// just sends the [`ResumePassthrough`] event in [`XrSetup`]
+pub struct EnablePassthroughStartup;
 
-#[inline]
-pub fn passthrough_layer_pause(mut xr_data_resource: ResMut<XrRenderData>) -> xr::Result<()> {
-    unsafe {
-        let passthrough_layer = &xr_data_resource.xr_passthrough_layer;
-        {
-            let passthrough_layer_locked = passthrough_layer.lock().unwrap();
-            cvt((xr_data_resource
-                .xr_instance
-                .exts()
-                .fb_passthrough
-                .unwrap()
-                .passthrough_layer_pause)(
-                *passthrough_layer_locked
-            ))?;
-        }
-        xr_data_resource.xr_passthrough_active = false;
-        bevy::log::info!("Paused passthrough layer");
-        Ok(())
+impl Plugin for EnablePassthroughStartup {
+    fn build(&self, app: &mut App) {
+        app.add_systems(XrSetup, |mut e: EventWriter<ResumePassthrough>| {
+            e.send_default();
+        });
     }
 }

src/prelude.rs

@@ -0,0 +1,15 @@
+use bevy::ecs::schedule::{IntoSystemConfigs, SystemConfigs};
+
+pub use crate::xr_init::schedules::XrSetup;
+use crate::xr_init::xr_only;
+pub use crate::xr_input::{QuatConv, Vec2Conv, Vec3Conv};
+
+pub trait XrSystems<Marker> {
+    fn xr_only(self) -> SystemConfigs;
+}
+
+impl<T: IntoSystemConfigs<M>, M> XrSystems<M> for T {
+    fn xr_only(self) -> SystemConfigs {
+        self.into_configs().run_if(xr_only())
+    }
+}

src/resource_macros.rs

@@ -1,7 +1,13 @@
 #[macro_export]
 macro_rules! xr_resource_wrapper {
     ($wrapper_type:ident, $xr_type:ty) => {
-        #[derive(Clone, bevy::prelude::Resource)]
+        #[derive(
+            Clone,
+            bevy::prelude::Resource,
+            bevy::prelude::Deref,
+            bevy::prelude::DerefMut,
+            bevy::render::extract_resource::ExtractResource,
+        )]
         pub struct $wrapper_type($xr_type);
 
         impl $wrapper_type {
@@ -10,13 +16,13 @@ macro_rules! xr_resource_wrapper {
             }
         }
 
-        impl std::ops::Deref for $wrapper_type {
-            type Target = $xr_type;
-
-            fn deref(&self) -> &Self::Target {
-                &self.0
-            }
-        }
+        // impl std::ops::Deref for $wrapper_type {
+        //     type Target = $xr_type;
+        //
+        //     fn deref(&self) -> &Self::Target {
+        //         &self.0
+        //     }
+        // }
 
         impl From<$xr_type> for $wrapper_type {
             fn from(value: $xr_type) -> Self {
@@ -26,10 +32,50 @@ macro_rules! xr_resource_wrapper {
     };
 }
 
+#[macro_export]
+macro_rules! xr_resource_wrapper_copy {
+    ($wrapper_type:ident, $xr_type:ty) => {
+        #[derive(
+            Clone,
+            Copy,
+            bevy::prelude::Resource,
+            bevy::prelude::Deref,
+            bevy::prelude::DerefMut,
+            bevy::render::extract_resource::ExtractResource,
+        )]
+        pub struct $wrapper_type($xr_type);
+
+        impl $wrapper_type {
+            pub fn new(value: $xr_type) -> Self {
+                Self(value)
+            }
+        }
+
+        // impl std::ops::Deref for $wrapper_type {
+        //     type Target = $xr_type;
+        //
+        //     fn deref(&self) -> &Self::Target {
+        //         &self.0
+        //     }
+        // }
+
+        impl From<$xr_type> for $wrapper_type {
+            fn from(value: $xr_type) -> Self {
+                Self::new(value)
+            }
+        }
+    };
+}
 #[macro_export]
 macro_rules! xr_arc_resource_wrapper {
     ($wrapper_type:ident, $xr_type:ty) => {
-        #[derive(Clone, bevy::prelude::Resource)]
+        #[derive(
+            Clone,
+            bevy::prelude::Resource,
+            bevy::prelude::Deref,
+            bevy::prelude::DerefMut,
+            bevy::render::extract_resource::ExtractResource,
+        )]
         pub struct $wrapper_type(std::sync::Arc<$xr_type>);
 
         impl $wrapper_type {
@@ -38,13 +84,41 @@ macro_rules! xr_arc_resource_wrapper {
             }
         }
 
-        impl std::ops::Deref for $wrapper_type {
-            type Target = $xr_type;
+        // impl std::ops::Deref for $wrapper_type {
+        //     type Target = $xr_type;
+        //
+        //     fn deref(&self) -> &Self::Target {
+        //         self.0.as_ref()
+        //     }
+        // }
 
-            fn deref(&self) -> &Self::Target {
-                self.0.as_ref()
+        impl From<$xr_type> for $wrapper_type {
+            fn from(value: $xr_type) -> Self {
+                Self::new(value)
             }
         }
+    };
+}
+
+#[macro_export]
+macro_rules! xr_no_clone_resource_wrapper {
+    ($wrapper_type:ident, $xr_type:ty) => {
+        #[derive(bevy::prelude::Resource, bevy::prelude::Deref, bevy::prelude::DerefMut)]
+        pub struct $wrapper_type($xr_type);
+
+        impl $wrapper_type {
+            pub fn new(value: $xr_type) -> Self {
+                Self(value)
+            }
+        }
+
+        // impl std::ops::Deref for $wrapper_type {
+        //     type Target = $xr_type;
+        //
+        //     fn deref(&self) -> &Self::Target {
+        //         &self.0
+        //     }
+        // }
 
         impl From<$xr_type> for $wrapper_type {
             fn from(value: $xr_type) -> Self {
@@ -55,4 +129,5 @@ macro_rules! xr_arc_resource_wrapper {
 }
 
 pub use xr_arc_resource_wrapper;
+pub use xr_no_clone_resource_wrapper;
 pub use xr_resource_wrapper;

src/resources.rs

@@ -1,26 +1,75 @@
+use std::ffi::c_void;
 use std::sync::atomic::AtomicBool;
 use std::sync::Mutex;
 
-use crate::passthrough::{Passthrough, PassthroughLayer};
+use crate::input::XrInput;
+use crate::passthrough::{CompositionLayerPassthrough, XrPassthroughLayer};
 use crate::resource_macros::*;
 use crate::xr::sys::CompositionLayerPassthroughFB;
 use crate::xr::{CompositionLayerBase, CompositionLayerFlags};
+use crate::{resource_macros::*, xr_resource_wrapper_copy};
 use bevy::prelude::*;
+use bevy::prelude::*;
+use bevy::render::extract_component::ExtractComponent;
+use bevy::render::extract_resource::{ExtractResource, ExtractResourcePlugin};
 use core::ptr;
 use openxr as xr;
 
 xr_resource_wrapper!(XrInstance, xr::Instance);
-xr_resource_wrapper!(XrSession, xr::Session<xr::AnyGraphics>);
-xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode);
-xr_resource_wrapper!(XrResolution, UVec2);
-xr_resource_wrapper!(XrFormat, wgpu::TextureFormat);
+xr_resource_wrapper_copy!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode);
+xr_resource_wrapper_copy!(XrResolution, UVec2);
+xr_resource_wrapper_copy!(XrFormat, wgpu::TextureFormat);
+xr_resource_wrapper_copy!(XrFrameState, xr::FrameState);
+xr_resource_wrapper!(XrViews, Vec<xr::View>);
 xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool);
-xr_arc_resource_wrapper!(XrFrameWaiter, Mutex<xr::FrameWaiter>);
 xr_arc_resource_wrapper!(XrSwapchain, Swapchain);
-xr_arc_resource_wrapper!(XrFrameState, Mutex<xr::FrameState>);
-xr_arc_resource_wrapper!(XrViews, Mutex<Vec<xr::View>>);
-xr_arc_resource_wrapper!(XrPassthrough, Mutex<xr::sys::PassthroughFB>);
-xr_arc_resource_wrapper!(XrPassthroughLayer, Mutex<xr::sys::PassthroughLayerFB>);
+xr_no_clone_resource_wrapper!(XrFrameWaiter, xr::FrameWaiter);
+
+#[derive(Clone, Resource, ExtractResource)]
+pub enum XrSession {
+    Vulkan(xr::Session<xr::Vulkan>),
+}
+
+impl std::ops::Deref for XrSession {
+    type Target = xr::Session<xr::AnyGraphics>;
+
+    fn deref(&self) -> &Self::Target {
+        // SAFTEY: should be fine i think -Schmarni
+        unsafe {
+            match self {
+                XrSession::Vulkan(sess) => std::mem::transmute(sess),
+            }
+        }
+    }
+}
+
+pub struct VulkanOXrSessionSetupInfo {
+    pub(crate) device_ptr: *const c_void,
+    pub(crate) physical_device_ptr: *const c_void,
+    pub(crate) vk_instance_ptr: *const c_void,
+    pub(crate) queue_family_index: u32,
+    pub(crate) xr_system_id: xr::SystemId,
+}
+
+pub enum OXrSessionSetupInfo {
+    Vulkan(VulkanOXrSessionSetupInfo),
+}
+
+pub struct XrResourcePlugin;
+
+impl Plugin for XrResourcePlugin {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(ExtractResourcePlugin::<XrResolution>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrFormat>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrSwapchain>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrFrameState>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrViews>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrInput>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrEnvironmentBlendMode>::default());
+        // app.add_plugins(ExtractResourcePlugin::<XrSessionRunning>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrSession>::default());
+    }
+}
 
 pub enum Swapchain {
     Vulkan(SwapchainInner<xr::Vulkan>),
@@ -64,7 +113,7 @@ impl Swapchain {
         stage: &xr::Space,
         resolution: UVec2,
         environment_blend_mode: xr::EnvironmentBlendMode,
-        passthrough_layer: Option<PassthroughLayer>,
+        passthrough_layer: Option<&XrPassthroughLayer>,
     ) -> xr::Result<()> {
         match self {
             Swapchain::Vulkan(swapchain) => swapchain.end(
@@ -85,6 +134,14 @@ pub struct SwapchainInner<G: xr::Graphics> {
     pub(crate) buffers: Vec<wgpu::Texture>,
     pub(crate) image_index: Mutex<usize>,
 }
+impl<G: xr::Graphics> Drop for SwapchainInner<G> {
+    fn drop(&mut self) {
+        for _ in 0..self.buffers.len() {
+            let v = self.buffers.remove(0);
+            Box::leak(Box::new(v));
+        }
+    }
+}
 
 impl<G: xr::Graphics> SwapchainInner<G> {
     fn begin(&self) -> xr::Result<()> {
@@ -133,7 +190,7 @@ impl<G: xr::Graphics> SwapchainInner<G> {
         stage: &xr::Space,
         resolution: UVec2,
         environment_blend_mode: xr::EnvironmentBlendMode,
-        passthrough_layer: Option<PassthroughLayer>,
+        passthrough_layer: Option<&XrPassthroughLayer>,
     ) -> xr::Result<()> {
         let rect = xr::Rect2Di {
             offset: xr::Offset2Di { x: 0, y: 0 },
@@ -143,7 +200,7 @@ impl<G: xr::Graphics> SwapchainInner<G> {
             },
         };
         let swapchain = self.handle.lock().unwrap();
-        if views.len() == 0 {
+        if views.is_empty() {
             warn!("views are len of 0");
             return Ok(());
         }
@@ -151,21 +208,11 @@ impl<G: xr::Graphics> SwapchainInner<G> {
             Some(pass) => {
                 //bevy::log::info!("Rendering with pass through");
 
-                let passthrough_layer = xr::sys::CompositionLayerPassthroughFB {
-                    ty: CompositionLayerPassthroughFB::TYPE,
-                    next: ptr::null(),
-                    flags: CompositionLayerFlags::UNPREMULTIPLIED_ALPHA,
-                    space: xr::sys::Space::NULL,
-                    layer_handle: pass.0,
-                };
-
                 self.stream.lock().unwrap().end(
                     predicted_display_time,
                     environment_blend_mode,
                     &[
-                        unsafe {
-                            &*(&passthrough_layer as *const _ as *const CompositionLayerBase<G>)
-                        },
+                        &CompositionLayerPassthrough::from_xr_passthrough_layer(pass),
                         &xr::CompositionLayerProjection::new()
                             .layer_flags(CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA)
                             .space(stage)
@@ -194,7 +241,7 @@ impl<G: xr::Graphics> SwapchainInner<G> {
             }
 
             None => {
-                bevy::log::info!("Rendering without pass through");
+                // bevy::log::info!("Rendering without pass through");
                 self.stream.lock().unwrap().end(
                     predicted_display_time,
                     environment_blend_mode,

src/xr_init.rs

@@ -1,352 +0,0 @@
-// Just a lot of code that is meant for something way more complex but hey.
-// maybe will work on that soon
-
-use std::sync::Arc;
-
-use bevy::{
-    ecs::schedule::{ExecutorKind, ScheduleLabel},
-    prelude::*,
-    render::{
-        extract_resource::{ExtractResource, ExtractResourcePlugin},
-        renderer::{self, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue},
-        settings::WgpuSettings,
-    },
-    window::RawHandleWrapper,
-};
-use wgpu::Instance;
-
-use crate::{
-    input::XrInput,
-    passthrough::{Passthrough, PassthroughLayer},
-    resources::{
-        XrEnvironmentBlendMode, XrFormat, XrFrameState, XrFrameWaiter, XrInstance, XrPassthrough,
-        XrPassthroughLayer, XrResolution, XrSession, XrSessionRunning, XrSwapchain, XrViews,
-    },
-};
-
-#[derive(Resource, Clone)]
-pub struct RenderCreationData {
-    pub device: RenderDevice,
-    pub queue: RenderQueue,
-    pub adapter_info: RenderAdapterInfo,
-    pub render_adapter: RenderAdapter,
-    pub instance: Arc<Instance>,
-}
-
-#[derive(Resource, Clone, ExtractResource)]
-pub struct XrRenderData {
-    pub xr_instance: XrInstance,
-    pub xr_session: XrSession,
-    pub xr_blend_mode: XrEnvironmentBlendMode,
-    pub xr_resolution: XrResolution,
-    pub xr_format: XrFormat,
-    pub xr_session_running: XrSessionRunning,
-    pub xr_frame_waiter: XrFrameWaiter,
-    pub xr_swapchain: XrSwapchain,
-    pub xr_input: XrInput,
-    pub xr_views: XrViews,
-    pub xr_frame_state: XrFrameState,
-    pub xr_passthrough_active: bool,
-    pub xr_passthrough: XrPassthrough,
-    pub xr_passthrough_layer: XrPassthroughLayer,
-}
-
-#[derive(Event, Clone, Copy, Debug)]
-pub enum XrEnableRequest {
-    TryEnable,
-    TryDisable,
-}
-#[derive(Resource, Event, Copy, Clone, PartialEq, Eq)]
-pub enum XrEnableStatus {
-    Enabled,
-    Disabled,
-    Waiting,
-}
-
-#[derive(Resource, Event, Copy, Clone, PartialEq, Eq, Debug)]
-pub enum XrNextEnabledState {
-    Enabled,
-    Disabled,
-}
-
-pub struct RenderRestartPlugin;
-
-#[derive(Resource)]
-pub struct ForceMain;
-
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrPreSetup;
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrSetup;
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrPrePostSetup;
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrPostSetup;
-
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrPreCleanup;
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrCleanup;
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrPostCleanup;
-
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrPreRenderUpdate;
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrRenderUpdate;
-#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct XrPostRenderUpdate;
-
-pub fn xr_only() -> impl FnMut(Option<Res<'_, XrEnableStatus>>) -> bool {
-    resource_exists_and_equals(XrEnableStatus::Enabled)
-}
-
-impl Plugin for RenderRestartPlugin {
-    fn build(&self, app: &mut App) {
-        add_schedules(app);
-        app.add_plugins(ExtractResourcePlugin::<XrRenderData>::default())
-            .insert_resource(ForceMain)
-            .add_event::<XrEnableRequest>()
-            .add_event::<XrEnableStatus>()
-            .add_systems(PostStartup, setup_xr.run_if(xr_only()))
-            .add_systems(
-                PostUpdate,
-                update_xr_stuff.run_if(on_event::<XrEnableRequest>()),
-            )
-            .add_systems(XrPreRenderUpdate, decide_next_xr_state)
-            .add_systems(XrPostRenderUpdate, clear_events)
-            .add_systems(
-                XrRenderUpdate,
-                (
-                    cleanup_xr.run_if(resource_exists_and_equals(XrNextEnabledState::Disabled)),
-                    // handle_xr_enable_requests,
-                    apply_deferred,
-                    setup_xr, /* .run_if(resource_exists_and_equals(XrEnableStatus::Enabled)) */
-                )
-                    .chain(),
-            )
-            .add_systems(XrCleanup, cleanup_oxr_session);
-    }
-}
-
-fn clear_events(mut events: ResMut<Events<XrEnableRequest>>) {
-    events.clear();
-}
-
-fn add_schedules(app: &mut App) {
-    let schedules = [
-        Schedule::new(XrPreSetup),
-        Schedule::new(XrSetup),
-        Schedule::new(XrPrePostSetup),
-        Schedule::new(XrPostSetup),
-        Schedule::new(XrPreRenderUpdate),
-        Schedule::new(XrRenderUpdate),
-        Schedule::new(XrPostRenderUpdate),
-        Schedule::new(XrPreCleanup),
-        Schedule::new(XrCleanup),
-        Schedule::new(XrPostCleanup),
-    ];
-    for mut schedule in schedules {
-        schedule.set_executor_kind(ExecutorKind::SingleThreaded);
-        schedule.set_apply_final_deferred(true);
-        app.add_schedule(schedule);
-    }
-}
-
-pub fn setup_xr(world: &mut World) {
-    world.run_schedule(XrPreSetup);
-    world.run_schedule(XrSetup);
-    world.run_schedule(XrPrePostSetup);
-    world.run_schedule(XrPostSetup);
-}
-fn cleanup_xr(world: &mut World) {
-    world.run_schedule(XrPreCleanup);
-    world.run_schedule(XrCleanup);
-    world.run_schedule(XrPostCleanup);
-}
-
-fn cleanup_oxr_session(xr_status: Option<Res<XrEnableStatus>>, session: Option<ResMut<XrSession>>) {
-    if let (Some(XrEnableStatus::Disabled), Some(s)) = (xr_status.map(|v| v.into_inner()), session)
-    {
-        s.into_inner().request_exit().unwrap();
-    }
-}
-
-pub fn update_xr_stuff(world: &mut World) {
-    world.run_schedule(XrPreRenderUpdate);
-    world.run_schedule(XrRenderUpdate);
-    world.run_schedule(XrPostRenderUpdate);
-}
-
-// fn handle_xr_enable_requests(
-//     primary_window: Query<&RawHandleWrapper, With<PrimaryWindow>>,
-//     mut commands: Commands,
-//     next_state: Res<XrNextEnabledState>,
-//     on_main: Option<NonSend<ForceMain>>,
-// ) {
-//     // Just to force this system onto the main thread because of unsafe code
-//     let _ = on_main;
-//
-//     commands.insert_resource(XrEnableStatus::Waiting);
-//     let (creation_data, xr_data) = match next_state.into_inner() {
-//         XrNextEnabledState::Enabled => {
-//             let (
-//                 device,
-//                 queue,
-//                 adapter_info,
-//                 render_adapter,
-//                 instance,
-//                 xr_instance,
-//                 session,
-//                 blend_mode,
-//                 resolution,
-//                 format,
-//                 session_running,
-//                 frame_waiter,
-//                 swapchain,
-//                 input,
-//                 views,
-//                 frame_state,
-//             ) = graphics::initialize_xr_graphics(primary_window.get_single().ok().cloned())
-//                 .unwrap();
-//
-//             commands.insert_resource(XrEnableStatus::Enabled);
-//             (
-//                 RenderCreationData {
-//                     device,
-//                     queue,
-//                     adapter_info,
-//                     render_adapter,
-//                     instance: Arc::new(instance),
-//                 },
-//                 Some(XrRenderData {
-//                     xr_instance,
-//                     xr_session: session,
-//                     xr_blend_mode: blend_mode,
-//                     xr_resolution: resolution,
-//                     xr_format: format,
-//                     xr_session_running: session_running,
-//                     xr_frame_waiter: frame_waiter,
-//                     xr_swapchain: swapchain,
-//                     xr_input: input,
-//                     xr_views: views,
-//                     xr_frame_state: frame_state,
-//                 }),
-//             )
-//         }
-//         XrNextEnabledState::Disabled => (
-//             init_non_xr_graphics(primary_window.get_single().ok().cloned()),
-//             None,
-//         ),
-//     };
-//
-//     commands.insert_resource(creation_data.device);
-//     commands.insert_resource(creation_data.queue);
-//     commands.insert_resource(RenderInstance(creation_data.instance));
-//     commands.insert_resource(creation_data.adapter_info);
-//     commands.insert_resource(creation_data.render_adapter);
-//
-//     if let Some(xr_data) = xr_data {
-//         // TODO: Remove this lib.rs:144
-//         commands.insert_resource(xr_data.clone());
-//
-//         commands.insert_resource(xr_data.xr_instance);
-//         commands.insert_resource(xr_data.xr_session);
-//         commands.insert_resource(xr_data.xr_blend_mode);
-//         commands.insert_resource(xr_data.xr_resolution);
-//         commands.insert_resource(xr_data.xr_format);
-//         commands.insert_resource(xr_data.xr_session_running);
-//         commands.insert_resource(xr_data.xr_frame_waiter);
-//         commands.insert_resource(xr_data.xr_input);
-//         commands.insert_resource(xr_data.xr_views);
-//         commands.insert_resource(xr_data.xr_frame_state);
-//         commands.insert_resource(xr_data.xr_swapchain);
-//     } else {
-//         commands.remove_resource::<XrRenderData>();
-//
-//         commands.remove_resource::<XrInstance>();
-//         commands.remove_resource::<XrSession>();
-//         commands.remove_resource::<XrEnvironmentBlendMode>();
-//         commands.remove_resource::<XrResolution>();
-//         commands.remove_resource::<XrFormat>();
-//         commands.remove_resource::<XrSessionRunning>();
-//         commands.remove_resource::<XrFrameWaiter>();
-//         commands.remove_resource::<XrSwapchain>();
-//         commands.remove_resource::<XrInput>();
-//         commands.remove_resource::<XrViews>();
-//         commands.remove_resource::<XrFrameState>();
-//     }
-// }
-
-fn decide_next_xr_state(
-    mut commands: Commands,
-    mut events: EventReader<XrEnableRequest>,
-    xr_status: Option<Res<XrEnableStatus>>,
-) {
-    info!("hm");
-    let request = match events.read().next() {
-        Some(v) => v,
-        None => return,
-    };
-    info!("ok");
-    match (request, xr_status.as_deref()) {
-        (XrEnableRequest::TryEnable, Some(XrEnableStatus::Enabled)) => {
-            info!("Xr Already Enabled! ignoring request");
-            return;
-        }
-        (XrEnableRequest::TryDisable, Some(XrEnableStatus::Disabled)) => {
-            info!("Xr Already Disabled! ignoring request");
-            return;
-        }
-        (_, Some(XrEnableStatus::Waiting)) => {
-            info!("Already Handling Request! ignoring request");
-            return;
-        }
-        _ => {}
-    }
-    let r = match request {
-        XrEnableRequest::TryEnable => XrNextEnabledState::Enabled,
-        XrEnableRequest::TryDisable => XrNextEnabledState::Disabled,
-    };
-    info!("{:#?}", r);
-    commands.insert_resource(r);
-}
-
-pub fn init_non_xr_graphics(primary_window: Option<RawHandleWrapper>) -> RenderCreationData {
-    let settings = WgpuSettings::default();
-
-    let async_renderer = async move {
-        let instance = wgpu::Instance::new(wgpu::InstanceDescriptor {
-            // Probably a bad idea unwraping here but on the other hand no backends
-            backends: settings.backends.unwrap(),
-            dx12_shader_compiler: settings.dx12_shader_compiler.clone(),
-        });
-        let surface = primary_window.map(|wrapper| unsafe {
-            // SAFETY: Plugins should be set up on the main thread.
-            let handle = wrapper.get_handle();
-            instance
-                .create_surface(&handle)
-                .expect("Failed to create wgpu surface")
-        });
-
-        let request_adapter_options = wgpu::RequestAdapterOptions {
-            power_preference: settings.power_preference,
-            compatible_surface: surface.as_ref(),
-            ..Default::default()
-        };
-
-        let (device, queue, adapter_info, render_adapter) =
-            renderer::initialize_renderer(&instance, &settings, &request_adapter_options).await;
-        debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
-        debug!("Configured wgpu adapter Features: {:#?}", device.features());
-        RenderCreationData {
-            device,
-            queue,
-            adapter_info,
-            render_adapter,
-            instance: Arc::new(instance),
-        }
-    };
-    // No need for wasm in bevy_oxr web xr would be a different crate
-    futures_lite::future::block_on(async_renderer)
-}

src/xr_init/mod.rs

@@ -0,0 +1,246 @@
+pub mod schedules;
+pub use schedules::*;
+
+use bevy::{
+    prelude::*,
+    render::{
+        camera::{ManualTextureView, ManualTextureViews},
+        extract_resource::{ExtractResource, ExtractResourcePlugin},
+        renderer::{RenderAdapter, RenderDevice, RenderInstance},
+        Render, RenderApp, RenderSet,
+    },
+    window::{PrimaryWindow, RawHandleWrapper},
+};
+
+use crate::{
+    clean_resources, graphics,
+    resources::{OXrSessionSetupInfo, XrFormat, XrInstance, XrResolution, XrSession, XrSwapchain},
+    LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE,
+};
+
+#[derive(Resource, Event, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource)]
+pub enum XrStatus {
+    NoInstance,
+    Enabled,
+    Enabling,
+    Disabled,
+    Disabling,
+}
+
+#[derive(
+    Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut,
+)]
+pub struct XrShouldRender(bool);
+#[derive(
+    Resource, Clone, Copy, PartialEq, Eq, Reflect, Debug, ExtractResource, Default, Deref, DerefMut,
+)]
+pub struct XrHasWaited(bool);
+
+pub struct XrEarlyInitPlugin;
+
+pub struct XrInitPlugin;
+
+pub fn xr_only() -> impl FnMut(Res<XrStatus>) -> bool {
+    resource_equals(XrStatus::Enabled)
+}
+pub fn xr_render_only() -> impl FnMut(Res<XrShouldRender>) -> bool {
+    resource_equals(XrShouldRender(true))
+}
+pub fn xr_after_wait_only() -> impl FnMut(Res<XrHasWaited>) -> bool {
+    resource_equals(XrHasWaited(true))
+}
+
+#[derive(Resource, Clone, Copy, ExtractResource)]
+pub struct CleanupRenderWorld;
+
+impl Plugin for XrEarlyInitPlugin {
+    fn build(&self, app: &mut App) {
+        add_schedules(app);
+        app.add_event::<SetupXrData>()
+            .add_event::<CleanupXrData>()
+            .add_event::<StartXrSession>()
+            .add_event::<EndXrSession>();
+    }
+}
+
+impl Plugin for XrInitPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(ExtractResourcePlugin::<XrStatus>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrShouldRender>::default());
+        app.add_plugins(ExtractResourcePlugin::<XrHasWaited>::default());
+        app.add_plugins(ExtractResourcePlugin::<CleanupRenderWorld>::default());
+        app.init_resource::<XrShouldRender>();
+        app.init_resource::<XrHasWaited>();
+        app.add_systems(PreUpdate, setup_xr.run_if(on_event::<SetupXrData>()))
+            .add_systems(PreUpdate, cleanup_xr.run_if(on_event::<CleanupXrData>()));
+        app.add_systems(
+            PostUpdate,
+            start_xr_session.run_if(on_event::<StartXrSession>()),
+        );
+        app.add_systems(
+            PostUpdate,
+            stop_xr_session.run_if(on_event::<EndXrSession>()),
+        );
+        app.add_systems(XrSetup, setup_manual_texture_views);
+        app.add_systems(XrCleanup, set_cleanup_res);
+        app.add_systems(PreUpdate, remove_cleanup_res.before(cleanup_xr));
+        let render_app = app.sub_app_mut(RenderApp);
+        render_app.add_systems(
+            Render,
+            remove_cleanup_res
+                .in_set(RenderSet::Cleanup)
+                .after(clean_resources),
+        );
+    }
+}
+
+#[derive(Resource, Clone, Copy, PartialEq, Eq, Default)]
+pub enum ExitAppOnSessionExit {
+    #[default]
+    /// Restart XrSession when session is lost
+    OnlyOnExit,
+    /// Always exit the app
+    Always,
+    /// Keep app open when XrSession wants to exit or is lost
+    Never,
+}
+
+pub struct StartSessionOnStartup;
+
+impl Plugin for StartSessionOnStartup {
+    fn build(&self, app: &mut App) {
+        app.add_systems(Startup, |mut event: EventWriter<StartXrSession>| {
+            event.send_default();
+        });
+    }
+}
+
+fn set_cleanup_res(mut commands: Commands) {
+    info!("Set Cleanup Res");
+    commands.insert_resource(CleanupRenderWorld);
+}
+fn remove_cleanup_res(mut commands: Commands) {
+    commands.remove_resource::<CleanupRenderWorld>();
+}
+
+fn setup_manual_texture_views(
+    mut manual_texture_views: ResMut<ManualTextureViews>,
+    swapchain: Res<XrSwapchain>,
+    xr_resolution: Res<XrResolution>,
+    xr_format: Res<XrFormat>,
+) {
+    info!("Creating Texture views");
+    let (left, right) = swapchain.get_render_views();
+    let left = ManualTextureView {
+        texture_view: left.into(),
+        size: **xr_resolution,
+        format: **xr_format,
+    };
+    let right = ManualTextureView {
+        texture_view: right.into(),
+        size: **xr_resolution,
+        format: **xr_format,
+    };
+    manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left);
+    manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right);
+}
+
+pub fn setup_xr(world: &mut World) {
+    info!("Pre XrPreSetup");
+    world.run_schedule(XrPreSetup);
+    info!("Post XrPreSetup");
+    world.run_schedule(XrSetup);
+    world.run_schedule(XrPrePostSetup);
+    world.run_schedule(XrPostSetup);
+    *world.resource_mut::<XrStatus>() = XrStatus::Enabled;
+}
+fn cleanup_xr(world: &mut World) {
+    world.run_schedule(XrPreCleanup);
+    world.run_schedule(XrCleanup);
+    world.run_schedule(XrPostCleanup);
+    *world.resource_mut::<XrStatus>() = XrStatus::Disabled;
+}
+
+#[derive(Event, Clone, Copy, Default)]
+pub struct StartXrSession;
+
+#[derive(Event, Clone, Copy, Default)]
+pub struct EndXrSession;
+
+#[derive(Event, Clone, Copy, Default)]
+pub(crate) struct SetupXrData;
+#[derive(Event, Clone, Copy, Default)]
+pub(crate) struct CleanupXrData;
+
+#[allow(clippy::too_many_arguments)]
+fn start_xr_session(
+    mut commands: Commands,
+    mut status: ResMut<XrStatus>,
+    instance: Res<XrInstance>,
+    primary_window: Query<&RawHandleWrapper, With<PrimaryWindow>>,
+    setup_info: NonSend<OXrSessionSetupInfo>,
+    render_device: Res<RenderDevice>,
+    render_adapter: Res<RenderAdapter>,
+    render_instance: Res<RenderInstance>,
+) {
+    info!("start Session");
+    match *status {
+        XrStatus::Disabled => {}
+        XrStatus::NoInstance => {
+            warn!("Trying to start OpenXR Session without instance, ignoring");
+            return;
+        }
+        XrStatus::Enabled | XrStatus::Enabling => {
+            warn!("Trying to start OpenXR Session while one already exists, ignoring");
+            return;
+        }
+        XrStatus::Disabling => {
+            warn!("Trying to start OpenXR Session while one is stopping, ignoring");
+            return;
+        }
+    }
+    let (
+        xr_session,
+        xr_resolution,
+        xr_format,
+        xr_session_running,
+        xr_frame_waiter,
+        xr_swapchain,
+        xr_input,
+        xr_views,
+        xr_frame_state,
+    ) = match graphics::start_xr_session(
+        primary_window.get_single().cloned().ok(),
+        &setup_info,
+        &instance,
+        &render_device,
+        &render_adapter,
+        &render_instance,
+    ) {
+        Ok(data) => data,
+        Err(err) => {
+            error!("Unable to start OpenXR Session: {}", err);
+            return;
+        }
+    };
+    commands.insert_resource(xr_session);
+    commands.insert_resource(xr_resolution);
+    commands.insert_resource(xr_format);
+    commands.insert_resource(xr_session_running);
+    commands.insert_resource(xr_frame_waiter);
+    commands.insert_resource(xr_swapchain);
+    commands.insert_resource(xr_input);
+    commands.insert_resource(xr_views);
+    commands.insert_resource(xr_frame_state);
+    *status = XrStatus::Enabling;
+}
+
+fn stop_xr_session(session: ResMut<XrSession>, mut status: ResMut<XrStatus>) {
+    match session.request_exit() {
+        Ok(_) => {}
+        Err(err) => {
+            error!("Error while trying to request session exit: {}", err)
+        }
+    }
+    *status = XrStatus::Disabling;
+}

src/xr_init/schedules.rs

@@ -0,0 +1,54 @@
+use bevy::{
+    app::App,
+    ecs::schedule::{ExecutorKind, Schedule, ScheduleLabel},
+};
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrPreSetup;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrSetup;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrPrePostSetup;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrPostSetup;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrPreCleanup;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrCleanup;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrPostCleanup;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrPreRenderUpdate;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrRenderUpdate;
+
+#[derive(Debug, ScheduleLabel, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct XrPostRenderUpdate;
+
+pub(super) fn add_schedules(app: &mut App) {
+    let schedules = [
+        Schedule::new(XrPreSetup),
+        Schedule::new(XrSetup),
+        Schedule::new(XrPrePostSetup),
+        Schedule::new(XrPostSetup),
+        Schedule::new(XrPreRenderUpdate),
+        Schedule::new(XrRenderUpdate),
+        Schedule::new(XrPostRenderUpdate),
+        Schedule::new(XrPreCleanup),
+        Schedule::new(XrCleanup),
+        Schedule::new(XrPostCleanup),
+    ];
+    for mut schedule in schedules {
+        schedule.set_executor_kind(ExecutorKind::SingleThreaded);
+        schedule.set_apply_final_deferred(true);
+        app.add_schedule(schedule);
+    }
+}

src/xr_input/actions.rs

@@ -6,21 +6,36 @@ use xr::{Action, Binding, Haptic, Posef, Vector2f};
 
 use crate::{
     resources::{XrInstance, XrSession},
-    xr_init::XrPrePostSetup,
+    xr_init::{xr_only, XrCleanup, XrPrePostSetup, XrPreSetup},
 };
 
 use super::oculus_touch::ActionSets;
 
 pub use xr::sys::NULL_PATH;
 
-pub struct OpenXrActionsPlugin;
-impl Plugin for OpenXrActionsPlugin {
+pub struct XrActionsPlugin;
+impl Plugin for XrActionsPlugin {
     fn build(&self, app: &mut App) {
-        app.insert_resource(SetupActionSets {
+        app.add_systems(PreUpdate, sync_actions.run_if(xr_only()));
+        app.add_systems(
+            XrPreSetup,
+            (insert_setup_action_sets, apply_deferred).chain(),
+        );
+        app.add_systems(XrPrePostSetup, setup_oxr_actions);
+        app.add_systems(XrCleanup, clean_actions);
+    }
+}
+
+fn insert_setup_action_sets(mut cmds: Commands) {
+    info!("WHAT?!");
+    cmds.insert_resource(SetupActionSets {
         sets: HashMap::new(),
     });
-        app.add_systems(XrPrePostSetup, setup_oxr_actions);
 }
+
+fn clean_actions(mut cmds: Commands) {
+    cmds.remove_resource::<ActionSets>();
+    cmds.remove_resource::<XrActionSets>();
 }
 
 #[inline(always)]
@@ -47,7 +62,6 @@ pub fn setup_oxr_actions(world: &mut World) {
     let right_path = instance.string_to_path("/user/hand/right").unwrap();
     let hands = [left_path, right_path];
 
-    let mut oxr_action_sets = Vec::new();
     let mut action_sets = XrActionSets { sets: default() };
     // let mut action_bindings: HashMap<&'static str, Vec<xr::Path>> = HashMap::new();
     let mut action_bindings: HashMap<
@@ -91,11 +105,11 @@ pub fn setup_oxr_actions(world: &mut World) {
                 }
             }
         }
-        oxr_action_sets.push(oxr_action_set);
+        // oxr_action_sets.push(oxr_action_set);
         action_sets.sets.insert(
             set_name,
             ActionSet {
-                // oxr_action_set,
+                oxr_action_set,
                 actions,
                 enabled: true,
             },
@@ -142,10 +156,15 @@ pub fn setup_oxr_actions(world: &mut World) {
             .expect("Unable to suggest interaction bindings!");
     }
     session
-        .attach_action_sets(&oxr_action_sets.iter().collect::<Vec<_>>())
+        .attach_action_sets(
+            &action_sets
+                .sets
+                .values()
+                .map(|set| &set.oxr_action_set)
+                .collect::<Vec<_>>(),
+        )
         .expect("Unable to attach action sets!");
 
-    world.insert_resource(ActionSets(oxr_action_sets));
     world.insert_resource(action_sets);
 }
 
@@ -206,7 +225,7 @@ impl SetupActionSet {
         for binding in bindings {
             self.actions
                 .get_mut(binding.action)
-                .ok_or(anyhow::anyhow!("Missing Action: {}", binding.action))
+                .ok_or(eyre::eyre!("Missing Action: {}", binding.action))
                 .unwrap()
                 .bindings
                 .entry(device_path)
@@ -257,6 +276,7 @@ pub struct ActionSet {
     // add functionality to enable/disable action sets
     enabled: bool,
     actions: HashMap<&'static str, TypedAction>,
+    oxr_action_set: xr::ActionSet,
 }
 
 #[derive(Resource)]
@@ -370,3 +390,20 @@ impl XrActionSets {
         }
     }
 }
+
+pub fn sync_actions(action_sets: Res<XrActionSets>, session: Res<XrSession>) {
+    let active_sets = action_sets
+        .sets
+        .values()
+        .filter_map(|set| {
+            if set.enabled {
+                Some(xr::ActiveActionSet::new(&set.oxr_action_set))
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+    if let Err(err) = session.sync_actions(&active_sets) {
+        warn!("OpenXR action sync error: {}", err);
+    }
+}

src/xr_input/controllers.rs

@@ -8,7 +8,3 @@ pub struct Handed<T> {
     pub left: T,
     pub right: T,
 }
-#[derive(Copy, Clone)]
-pub enum XrControllerType {
-    OculusTouch,
-}

src/xr_input/debug_gizmos.rs

@@ -1,5 +1,6 @@
 use bevy::ecs::schedule::IntoSystemConfigs;
 use bevy::log::{debug, info};
+use bevy::math::primitives::Direction3d;
 use bevy::prelude::{
     Color, Gizmos, GlobalTransform, Plugin, Quat, Query, Res, Transform, Update, Vec2, Vec3, With,
     Without,
@@ -91,7 +92,7 @@ pub fn draw_gizmos(
     //     }
     // }
     //lock frame
-    let frame_state = *frame_state.lock().unwrap();
+    // let frame_state = *frame_state.lock().unwrap();
     //get controller
     let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
     let root = tracking_root_query.get_single();
@@ -104,7 +105,7 @@ pub fn draw_gizmos(
                         y: 0.01,
                         z: 0.0,
                     },
-                Vec3::Y,
+                Direction3d::Y,
                 0.2,
                 Color::RED,
             );
@@ -168,7 +169,7 @@ fn draw_hand_gizmo(
             //draw face
             gizmos.circle(
                 face_translation_vec3,
-                face_quat_normal,
+                Direction3d::new_unchecked(face_quat_normal),
                 0.04,
                 Color::YELLOW_GREEN,
             );
@@ -185,7 +186,12 @@ fn draw_hand_gizmo(
             let b_offset_quat = face_quat;
             let b_translation_vec3 =
                 face_translation_vec3 + b_offset_quat.mul_vec3(Vec3::new(0.025, -0.01, 0.0));
-            gizmos.circle(b_translation_vec3, face_quat_normal, 0.0075, b_color);
+            gizmos.circle(
+                b_translation_vec3,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.0075,
+                b_color,
+            );
 
             //button a
             let mut a_color = off_color;
@@ -199,7 +205,12 @@ fn draw_hand_gizmo(
             let a_offset_quat = face_quat;
             let a_translation_vec3 =
                 face_translation_vec3 + a_offset_quat.mul_vec3(Vec3::new(0.025, 0.01, 0.0));
-            gizmos.circle(a_translation_vec3, face_quat_normal, 0.0075, a_color);
+            gizmos.circle(
+                a_translation_vec3,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.0075,
+                a_color,
+            );
 
             //joystick
             let joystick_offset_quat = face_quat;
@@ -211,7 +222,12 @@ fn draw_hand_gizmo(
             }
 
             //base
-            gizmos.circle(joystick_base_vec, face_quat_normal, 0.014, joystick_color);
+            gizmos.circle(
+                joystick_base_vec,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.014,
+                joystick_color,
+            );
 
             let stick = controller.thumbstick(Hand::Left);
             let input = Vec3::new(stick.x, -stick.y, 0.0);
@@ -219,7 +235,12 @@ fn draw_hand_gizmo(
                 + joystick_offset_quat.mul_vec3(Vec3::new(-0.02, 0.0, -0.01))
                 + joystick_offset_quat.mul_vec3(input * 0.01);
             //top
-            gizmos.circle(joystick_top_vec, face_quat_normal, 0.005, joystick_color);
+            gizmos.circle(
+                joystick_top_vec,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.005,
+                joystick_color,
+            );
 
             //trigger
             let trigger_state = controller.trigger(Hand::Left);
@@ -277,7 +298,7 @@ fn draw_hand_gizmo(
             //draw face
             gizmos.circle(
                 face_translation_vec3,
-                face_quat_normal,
+                Direction3d::new_unchecked(face_quat_normal),
                 0.04,
                 Color::YELLOW_GREEN,
             );
@@ -294,7 +315,12 @@ fn draw_hand_gizmo(
             let b_offset_quat = face_quat;
             let b_translation_vec3 =
                 face_translation_vec3 + b_offset_quat.mul_vec3(Vec3::new(-0.025, -0.01, 0.0));
-            gizmos.circle(b_translation_vec3, face_quat_normal, 0.0075, b_color);
+            gizmos.circle(
+                b_translation_vec3,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.0075,
+                b_color,
+            );
 
             //button a
             let mut a_color = off_color;
@@ -308,7 +334,12 @@ fn draw_hand_gizmo(
             let a_offset_quat = face_quat;
             let a_translation_vec3 =
                 face_translation_vec3 + a_offset_quat.mul_vec3(Vec3::new(-0.025, 0.01, 0.0));
-            gizmos.circle(a_translation_vec3, face_quat_normal, 0.0075, a_color);
+            gizmos.circle(
+                a_translation_vec3,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.0075,
+                a_color,
+            );
 
             //joystick time
             let joystick_offset_quat = face_quat;
@@ -320,7 +351,12 @@ fn draw_hand_gizmo(
             }
 
             //base
-            gizmos.circle(joystick_base_vec, face_quat_normal, 0.014, joystick_color);
+            gizmos.circle(
+                joystick_base_vec,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.014,
+                joystick_color,
+            );
 
             let stick = controller.thumbstick(Hand::Right);
             let input = Vec3::new(stick.x, -stick.y, 0.0);
@@ -328,7 +364,12 @@ fn draw_hand_gizmo(
                 + joystick_offset_quat.mul_vec3(Vec3::new(0.02, 0.0, -0.01))
                 + joystick_offset_quat.mul_vec3(input * 0.01);
             //top
-            gizmos.circle(joystick_top_vec, face_quat_normal, 0.005, joystick_color);
+            gizmos.circle(
+                joystick_top_vec,
+                Direction3d::new_unchecked(face_quat_normal),
+                0.005,
+                joystick_color,
+            );
 
             //trigger
             let trigger_state = controller.trigger(Hand::Right);

src/xr_input/hand_poses.rs

@@ -136,8 +136,7 @@ pub fn get_simulated_open_hand_transforms(hand: Hand) -> [Transform; 26] {
             z: 0.01,
         },
     ];
-    let result = bones_to_transforms(test_hand_bones, hand);
-    return result;
+    bones_to_transforms(test_hand_bones, hand)
 }
 
 fn bones_to_transforms(hand_bones: [Vec3; 26], hand: Hand) -> [Transform; 26] {

src/xr_input/hands/common.rs

@@ -1,6 +1,10 @@
-use bevy::prelude::{
+use bevy::{
+    core::Name,
+    prelude::{
         default, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, Plugin, PostUpdate,
         Query, Resource, SpatialBundle, Startup, Transform,
+    },
+    transform::components::GlobalTransform,
 };
 
 use crate::xr_input::{trackers::OpenXRTracker, Hand};
@@ -8,14 +12,14 @@ use crate::xr_input::{trackers::OpenXRTracker, Hand};
 use super::{BoneTrackingStatus, HandBone};
 
 /// add debug renderer for controllers
-#[derive(Default)]
-pub struct OpenXrHandInput;
-
-impl Plugin for OpenXrHandInput {
-    fn build(&self, app: &mut bevy::prelude::App) {
-        app.add_systems(Startup, spawn_hand_entities);
-    }
-}
+// #[derive(Default)]
+// pub struct OpenXrHandInput;
+//
+// impl Plugin for OpenXrHandInput {
+//     fn build(&self, app: &mut bevy::prelude::App) {
+//         app.add_systems(Startup, spawn_hand_entities);
+//     }
+// }
 
 /// add debug renderer for controllers
 #[derive(Default)]
@@ -161,75 +165,46 @@ pub fn spawn_hand_entities(mut commands: Commands) {
         for bone in bones.iter() {
             let boneid = commands
                 .spawn((
+                    Name::new(format!("{:?} {:?}", hand, bone)),
                     SpatialBundle::default(),
-                    bone.clone(),
+                    *bone,
                     OpenXRTracker,
-                    hand.clone(),
-                    BoneTrackingStatus::Emulated,
+                    *hand,
+                    BoneTrackingStatus::Tracked,
                     HandBoneRadius(0.1),
                 ))
                 .id();
-            match hand {
-                Hand::Left => match bone {
-                    HandBone::Palm => hand_resource.left.palm = boneid,
-                    HandBone::Wrist => hand_resource.left.wrist = boneid,
-                    HandBone::ThumbMetacarpal => hand_resource.left.thumb.metacarpal = boneid,
-                    HandBone::ThumbProximal => hand_resource.left.thumb.proximal = boneid,
-                    HandBone::ThumbDistal => hand_resource.left.thumb.distal = boneid,
-                    HandBone::ThumbTip => hand_resource.left.thumb.tip = boneid,
-                    HandBone::IndexMetacarpal => hand_resource.left.index.metacarpal = boneid,
-                    HandBone::IndexProximal => hand_resource.left.index.proximal = boneid,
-                    HandBone::IndexIntermediate => hand_resource.left.index.intermediate = boneid,
-                    HandBone::IndexDistal => hand_resource.left.index.distal = boneid,
-                    HandBone::IndexTip => hand_resource.left.index.tip = boneid,
-                    HandBone::MiddleMetacarpal => hand_resource.left.middle.metacarpal = boneid,
-                    HandBone::MiddleProximal => hand_resource.left.middle.proximal = boneid,
-                    HandBone::MiddleIntermediate => hand_resource.left.middle.intermediate = boneid,
-                    HandBone::MiddleDistal => hand_resource.left.middle.distal = boneid,
-                    HandBone::MiddleTip => hand_resource.left.middle.tip = boneid,
-                    HandBone::RingMetacarpal => hand_resource.left.ring.metacarpal = boneid,
-                    HandBone::RingProximal => hand_resource.left.ring.proximal = boneid,
-                    HandBone::RingIntermediate => hand_resource.left.ring.intermediate = boneid,
-                    HandBone::RingDistal => hand_resource.left.ring.distal = boneid,
-                    HandBone::RingTip => hand_resource.left.ring.tip = boneid,
-                    HandBone::LittleMetacarpal => hand_resource.left.little.metacarpal = boneid,
-                    HandBone::LittleProximal => hand_resource.left.little.proximal = boneid,
-                    HandBone::LittleIntermediate => hand_resource.left.little.intermediate = boneid,
-                    HandBone::LittleDistal => hand_resource.left.little.distal = boneid,
-                    HandBone::LittleTip => hand_resource.left.little.tip = boneid,
-                },
-                Hand::Right => match bone {
-                    HandBone::Palm => hand_resource.right.palm = boneid,
-                    HandBone::Wrist => hand_resource.right.wrist = boneid,
-                    HandBone::ThumbMetacarpal => hand_resource.right.thumb.metacarpal = boneid,
-                    HandBone::ThumbProximal => hand_resource.right.thumb.proximal = boneid,
-                    HandBone::ThumbDistal => hand_resource.right.thumb.distal = boneid,
-                    HandBone::ThumbTip => hand_resource.right.thumb.tip = boneid,
-                    HandBone::IndexMetacarpal => hand_resource.right.index.metacarpal = boneid,
-                    HandBone::IndexProximal => hand_resource.right.index.proximal = boneid,
-                    HandBone::IndexIntermediate => hand_resource.right.index.intermediate = boneid,
-                    HandBone::IndexDistal => hand_resource.right.index.distal = boneid,
-                    HandBone::IndexTip => hand_resource.right.index.tip = boneid,
-                    HandBone::MiddleMetacarpal => hand_resource.right.middle.metacarpal = boneid,
-                    HandBone::MiddleProximal => hand_resource.right.middle.proximal = boneid,
-                    HandBone::MiddleIntermediate => {
-                        hand_resource.right.middle.intermediate = boneid
-                    }
-                    HandBone::MiddleDistal => hand_resource.right.middle.distal = boneid,
-                    HandBone::MiddleTip => hand_resource.right.middle.tip = boneid,
-                    HandBone::RingMetacarpal => hand_resource.right.ring.metacarpal = boneid,
-                    HandBone::RingProximal => hand_resource.right.ring.proximal = boneid,
-                    HandBone::RingIntermediate => hand_resource.right.ring.intermediate = boneid,
-                    HandBone::RingDistal => hand_resource.right.ring.distal = boneid,
-                    HandBone::RingTip => hand_resource.right.ring.tip = boneid,
-                    HandBone::LittleMetacarpal => hand_resource.right.little.metacarpal = boneid,
-                    HandBone::LittleProximal => hand_resource.right.little.proximal = boneid,
-                    HandBone::LittleIntermediate => {
-                        hand_resource.right.little.intermediate = boneid
-                    }
-                    HandBone::LittleDistal => hand_resource.right.little.distal = boneid,
-                    HandBone::LittleTip => hand_resource.right.little.tip = boneid,
-                },
+            let hand_res = match hand {
+                Hand::Left => &mut hand_resource.left,
+                Hand::Right => &mut hand_resource.right,
+            };
+            match bone {
+                HandBone::Palm => hand_res.palm = boneid,
+                HandBone::Wrist => hand_res.wrist = boneid,
+                HandBone::ThumbMetacarpal => hand_res.thumb.metacarpal = boneid,
+                HandBone::ThumbProximal => hand_res.thumb.proximal = boneid,
+                HandBone::ThumbDistal => hand_res.thumb.distal = boneid,
+                HandBone::ThumbTip => hand_res.thumb.tip = boneid,
+                HandBone::IndexMetacarpal => hand_res.index.metacarpal = boneid,
+                HandBone::IndexProximal => hand_res.index.proximal = boneid,
+                HandBone::IndexIntermediate => hand_res.index.intermediate = boneid,
+                HandBone::IndexDistal => hand_res.index.distal = boneid,
+                HandBone::IndexTip => hand_res.index.tip = boneid,
+                HandBone::MiddleMetacarpal => hand_res.middle.metacarpal = boneid,
+                HandBone::MiddleProximal => hand_res.middle.proximal = boneid,
+                HandBone::MiddleIntermediate => hand_res.middle.intermediate = boneid,
+                HandBone::MiddleDistal => hand_res.middle.distal = boneid,
+                HandBone::MiddleTip => hand_res.middle.tip = boneid,
+                HandBone::RingMetacarpal => hand_res.ring.metacarpal = boneid,
+                HandBone::RingProximal => hand_res.ring.proximal = boneid,
+                HandBone::RingIntermediate => hand_res.ring.intermediate = boneid,
+                HandBone::RingDistal => hand_res.ring.distal = boneid,
+                HandBone::RingTip => hand_res.ring.tip = boneid,
+                HandBone::LittleMetacarpal => hand_res.little.metacarpal = boneid,
+                HandBone::LittleProximal => hand_res.little.proximal = boneid,
+                HandBone::LittleIntermediate => hand_res.little.intermediate = boneid,
+                HandBone::LittleDistal => hand_res.little.distal = boneid,
+                HandBone::LittleTip => hand_res.little.tip = boneid,
             }
         }
     }
@@ -241,16 +216,12 @@ pub struct HandBoneRadius(pub f32);
 
 pub fn draw_hand_entities(
     mut gizmos: Gizmos,
-    query: Query<(&Transform, &HandBone, &HandBoneRadius)>,
+    query: Query<(&GlobalTransform, &HandBone, &HandBoneRadius)>,
 ) {
     for (transform, hand_bone, hand_bone_radius) in query.iter() {
         let (_, color) = get_bone_gizmo_style(hand_bone);
-        gizmos.sphere(
-            transform.translation,
-            transform.rotation,
-            hand_bone_radius.0,
-            color,
-        );
+        let (_, rotation, translation) = transform.to_scale_rotation_translation();
+        gizmos.sphere(translation, rotation, hand_bone_radius.0, color);
     }
 }
 

src/xr_input/hands/emulated.rs

@@ -92,8 +92,6 @@ fn setup_hand_emulation_action_set(mut action_sets: ResMut<SetupActionSets>) {
     suggest_oculus_touch_profile(action_set);
 }
 
-pub struct EmulatedHandPoseData {}
-
 fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) {
     action_set.suggest_binding(
         "/interaction_profiles/oculus/touch_controller",
@@ -131,7 +129,6 @@ pub(crate) fn update_hand_skeleton_from_emulated(
     action_sets: Res<XrActionSets>,
     left_controller_transform: Query<&Transform, With<OpenXRLeftController>>,
     right_controller_transform: Query<&Transform, With<OpenXRRightController>>,
-    tracking_root_transform: Query<&Transform, With<OpenXRTrackingRoot>>,
     mut bones: Query<
         (
             &mut Transform,
@@ -226,7 +223,6 @@ pub(crate) fn update_hand_skeleton_from_emulated(
             },
         }
     }
-    let trt = tracking_root_transform.single();
     for (mut t, bone, hand, status, mut radius) in bones.iter_mut() {
         match status {
             BoneTrackingStatus::Emulated => {}
@@ -238,9 +234,9 @@ pub(crate) fn update_hand_skeleton_from_emulated(
             Hand::Left => 0,
             Hand::Right => 1,
         }][bone.get_index_from_bone()];
-        *t = t.with_scale(trt.scale);
-        *t = t.with_rotation(trt.rotation * t.rotation);
-        *t = t.with_translation(trt.transform_point(t.translation));
+        // *t = t.with_scale(trt.scale);
+        // *t = t.with_rotation(trt.rotation * t.rotation);
+        // *t = t.with_translation(trt.transform_point(t.translation));
     }
 }
 pub fn update_hand_bones_emulated(

src/xr_input/hands/hand_tracking.rs

@@ -6,7 +6,7 @@ use crate::{
     input::XrInput,
     resources::{XrFrameState, XrSession},
     xr_init::xr_only,
-    xr_input::{hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv, Vec3Conv},
+    xr_input::{hands::HandBone, Hand, QuatConv, Vec3Conv},
 };
 
 use super::BoneTrackingStatus;
@@ -63,6 +63,7 @@ pub struct HandJoint {
     pub radius: f32,
 }
 
+#[derive(Debug)]
 pub struct HandJoints {
     inner: [HandJoint; 26],
 }
@@ -87,7 +88,7 @@ impl<'a> HandTrackingRef<'a> {
                     Hand::Left => &self.tracking.left_hand,
                     Hand::Right => &self.tracking.right_hand,
                 },
-                self.frame_state.lock().unwrap().predicted_display_time,
+                self.frame_state.predicted_display_time,
             )
             .unwrap()
             .map(|joints| {
@@ -158,7 +159,6 @@ pub fn update_hand_bones(
     hand_tracking: Option<Res<HandTrackingData>>,
     xr_input: Res<XrInput>,
     xr_frame_state: Res<XrFrameState>,
-    root_query: Query<(&Transform, With<OpenXRTrackingRoot>, Without<HandBone>)>,
     mut bones: Query<(
         &mut Transform,
         &Hand,
@@ -174,9 +174,12 @@ pub fn update_hand_bones(
             return;
         }
     };
-    let (root_transform, _, _) = root_query.get_single().unwrap();
     let left_hand_data = hand_ref.get_poses(Hand::Left);
     let right_hand_data = hand_ref.get_poses(Hand::Right);
+    if left_hand_data.is_none() || right_hand_data.is_none() {
+        error!("something is very wrong for hand_tracking!! doesn't have data for both hands!");
+    }
+
     bones
         .par_iter_mut()
         .for_each(|(mut transform, hand, bone, mut radius, mut status)| {
@@ -194,7 +197,7 @@ pub fn update_hand_bones(
             let bone_data = match (hand, &left_hand_data, &right_hand_data) {
                 (Hand::Left, Some(data), _) => data.get_joint(*bone),
                 (Hand::Right, _, Some(data)) => data.get_joint(*bone),
-                _ => {
+                (hand, left_data, right_data) => {
                     *status = BoneTrackingStatus::Emulated;
                     return;
                 }
@@ -203,8 +206,7 @@ pub fn update_hand_bones(
                 *status = BoneTrackingStatus::Tracked;
             }
             radius.0 = bone_data.radius;
-            *transform = transform
-                .with_translation(root_transform.transform_point(bone_data.position))
-                .with_rotation(root_transform.rotation * bone_data.orientation)
+            transform.translation = bone_data.position;
+            transform.rotation = bone_data.orientation;
         });
 }

src/xr_input/hands/mod.rs

@@ -1,19 +1,64 @@
-use bevy::{app::PluginGroupBuilder, prelude::*};
+use bevy::prelude::*;
+use openxr::FormFactor;
 
-use self::{emulated::HandEmulationPlugin, hand_tracking::HandTrackingPlugin};
+use crate::{
+    resources::{XrInstance, XrSession},
+    xr_init::{XrCleanup, XrPreSetup, XrSetup},
+};
+
+use self::{
+    common::{spawn_hand_entities, HandBoneRadius, HandsResource},
+    hand_tracking::{DisableHandTracking, HandTrackingData},
+};
+
+use super::{trackers::OpenXRTracker, Hand};
 
 pub mod common;
 pub mod emulated;
 pub mod hand_tracking;
 
-pub struct XrHandPlugins;
+pub struct HandPlugin;
 
-impl PluginGroup for XrHandPlugins {
-    fn build(self) -> PluginGroupBuilder {
-        PluginGroupBuilder::start::<Self>()
-            .add(HandTrackingPlugin)
-            .add(HandEmulationPlugin)
-            .build()
+impl Plugin for HandPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(XrPreSetup, check_for_handtracking);
+        app.add_systems(XrSetup, spawn_hand_entities);
+        app.add_systems(XrCleanup, despawn_hand_entities);
+    }
+}
+
+#[allow(clippy::type_complexity)]
+fn despawn_hand_entities(
+    mut commands: Commands,
+    hand_entities: Query<
+        Entity,
+        (
+            With<OpenXRTracker>,
+            With<HandBone>,
+            With<BoneTrackingStatus>,
+        ),
+    >,
+) {
+    for e in &hand_entities {
+        commands.entity(e).despawn_recursive();
+    }
+    commands.remove_resource::<HandsResource>()
+}
+
+fn check_for_handtracking(
+    mut commands: Commands,
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+) {
+    let hands = instance.exts().ext_hand_tracking.is_some()
+        && instance
+            .supports_hand_tracking(instance.system(FormFactor::HEAD_MOUNTED_DISPLAY).unwrap())
+            .is_ok_and(|v| v);
+    if hands {
+        info!("handtracking!");
+        commands.insert_resource(HandTrackingData::new(&session).unwrap());
+    } else {
+        commands.insert_resource(DisableHandTracking::Both);
     }
 }
 
@@ -54,21 +99,17 @@ pub enum HandBone {
 }
 impl HandBone {
     pub fn is_finger(&self) -> bool {
-        match &self {
-            HandBone::Wrist => false,
-            HandBone::Palm => false,
-            _ => true,
-        }
+        !matches!(self, HandBone::Wrist | HandBone::Palm)
     }
     pub fn is_metacarpal(&self) -> bool {
-        match &self {
-            HandBone::ThumbMetacarpal => true,
-            HandBone::IndexMetacarpal => true,
-            HandBone::MiddleMetacarpal => true,
-            HandBone::RingMetacarpal => true,
-            HandBone::LittleTip => true,
-            _ => false,
-        }
+        matches!(
+            self,
+            HandBone::ThumbMetacarpal
+                | HandBone::IndexMetacarpal
+                | HandBone::MiddleMetacarpal
+                | HandBone::RingMetacarpal
+                | HandBone::LittleTip
+        )
     }
     pub const fn get_all_bones() -> [HandBone; 26] {
         [

src/xr_input/interactions.rs

@@ -1,6 +1,6 @@
 use std::f32::consts::PI;
 
-use bevy::log::info;
+use bevy::log::{info, warn};
 use bevy::prelude::{
     Color, Component, Entity, Event, EventReader, EventWriter, Gizmos, GlobalTransform, Quat,
     Query, Transform, Vec3, With, Without,
@@ -93,9 +93,12 @@ pub fn draw_interaction_gizmos(
         ),
         Without<XRInteractable>,
     >,
-    tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
+    tracking_root_query: Query<&mut Transform, With<OpenXRTrackingRoot>>,
 ) {
-    let root = tracking_root_query.get_single().unwrap().0;
+    let Ok(root) = tracking_root_query.get_single() else {
+        warn!("no or more than one tracking root");
+        return;
+    };
     for (global_transform, interactable_state) in interactable_query.iter() {
         let transform = global_transform.compute_transform();
         let color = match interactable_state {
@@ -137,7 +140,7 @@ pub fn draw_interaction_gizmos(
                     };
                     gizmos.ray(
                         root.translation + root.rotation.mul_vec3(aim.0.translation),
-                        root.rotation.mul_vec3(aim.0.forward()),
+                        root.rotation.mul_vec3(*aim.0.forward()),
                         color,
                     );
                 }
@@ -229,7 +232,7 @@ pub fn interactions(
         ),
         Without<XRInteractable>,
     >,
-    tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
+    tracking_root_query: Query<&mut Transform, With<OpenXRTrackingRoot>>,
     mut writer: EventWriter<InteractionEvent>,
 ) {
     for (xr_interactable_global_transform, interactable_entity) in interactable_query.iter() {
@@ -280,12 +283,12 @@ pub fn interactions(
                     let center = sphere_transform.translation;
                     let radius: f32 = 0.1;
                     //I hate this but the aim pose needs the root for now
-                    let root = tracking_root_query.get_single().unwrap().0;
+                    let root = tracking_root_query.get_single().unwrap();
                     match aim {
                         Some(aim) => {
                             let ray_origin =
                                 root.translation + root.rotation.mul_vec3(aim.0.translation);
-                            let ray_dir = root.rotation.mul_vec3(aim.0.forward());
+                            let ray_dir = root.rotation.mul_vec3(*aim.0.forward());
 
                             if ray_sphere_intersection(
                                 center,

src/xr_input/mod.rs

@@ -10,124 +10,92 @@ pub mod trackers;
 pub mod xr_camera;
 
 use crate::resources::{XrInstance, XrSession};
-use crate::xr_begin_frame;
-use crate::xr_init::{xr_only, XrPostSetup, XrSetup, XrPreSetup};
-use crate::xr_input::controllers::XrControllerType;
+use crate::xr_init::{xr_only, XrCleanup, XrPostSetup, XrPreSetup, XrSetup};
 use crate::xr_input::oculus_touch::setup_oculus_controller;
 use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle};
+use crate::{locate_views, xr_wait_frame};
 use bevy::app::{App, PostUpdate, Startup};
 use bevy::ecs::entity::Entity;
 use bevy::ecs::query::With;
 use bevy::ecs::system::Query;
+use bevy::hierarchy::DespawnRecursiveExt;
 use bevy::log::{info, warn};
 use bevy::math::Vec2;
 use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource};
 use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3};
 use bevy::render::camera::CameraProjectionPlugin;
+use bevy::render::extract_component::ExtractComponentPlugin;
 use bevy::render::view::{update_frusta, VisibilitySystems};
 use bevy::transform::TransformSystem;
 use bevy::utils::HashMap;
 use openxr::Binding;
 
-use self::actions::{setup_oxr_actions, OpenXrActionsPlugin};
-use self::oculus_touch::{post_action_setup_oculus_controller, ActionSets, init_subaction_path};
+use self::actions::{setup_oxr_actions, XrActionsPlugin};
+use self::oculus_touch::{
+    init_subaction_path, post_action_setup_oculus_controller, ActionSets, OculusController,
+};
 use self::trackers::{
     adopt_open_xr_trackers, update_open_xr_controllers, OpenXRLeftEye, OpenXRRightEye,
     OpenXRTrackingRoot,
 };
+use self::xr_camera::{/* GlobalTransformExtract, TransformExtract, */ XrCamera};
 
 #[derive(Copy, Clone)]
-pub struct OpenXrInput {
-    pub controller_type: XrControllerType,
-}
+pub struct XrInputPlugin;
 #[derive(Clone, Copy, Debug, Ord, PartialOrd, Eq, PartialEq, Component)]
 pub enum Hand {
     Left,
     Right,
 }
 
-impl OpenXrInput {
-    pub fn new(controller_type: XrControllerType) -> Self {
-        Self { controller_type }
-    }
-}
-
-impl Plugin for OpenXrInput {
+impl Plugin for XrInputPlugin {
     fn build(&self, app: &mut App) {
-        app.add_plugins(CameraProjectionPlugin::<XRProjection>::default());
-        app.add_plugins(OpenXrActionsPlugin);
         app.add_systems(XrPostSetup, post_action_setup_oculus_controller);
-        match self.controller_type {
-            XrControllerType::OculusTouch => {
         app.add_systems(XrSetup, setup_oculus_controller);
-            }
-        }
+        app.add_systems(XrCleanup, cleanup_oculus_controller);
         //adopt any new trackers
         app.add_systems(PreUpdate, adopt_open_xr_trackers.run_if(xr_only()));
-        app.add_systems(PreUpdate, action_set_system.run_if(xr_only()));
-        app.add_systems(
-            PreUpdate,
-            xr_camera_head_sync.run_if(xr_only()).after(xr_begin_frame),
-        );
+        // app.add_systems(PreUpdate, action_set_system.run_if(xr_only()));
         //update controller trackers
         app.add_systems(Update, update_open_xr_controllers.run_if(xr_only()));
-        app.add_systems(
-            PostUpdate,
-            update_frusta::<XRProjection>
-                .after(TransformSystem::TransformPropagate)
-                .before(VisibilitySystems::UpdatePerspectiveFrusta),
-        );
         app.add_systems(XrPreSetup, init_subaction_path);
-        app.add_systems(XrSetup, setup_xr_cameras);
+        app.add_systems(XrSetup, setup_xr_root);
+        app.add_systems(XrCleanup, cleanup_xr_root);
     }
 }
 
-#[derive(Deref, DerefMut, Resource)]
-pub struct InteractionProfileBindings(pub HashMap<&'static str, Vec<Binding<'static>>>);
-
-fn setup_binding_recommendations(
-    mut commands: Commands,
-    instance: Res<XrInstance>,
-    bindings: Res<InteractionProfileBindings>,
-) {
-    commands.remove_resource::<InteractionProfileBindings>();
+fn cleanup_oculus_controller(mut commands: Commands) {
+    commands.remove_resource::<OculusController>();
 }
 
-fn setup_xr_cameras(
+fn cleanup_xr_root(
     mut commands: Commands,
     tracking_root_query: Query<Entity, With<OpenXRTrackingRoot>>,
 ) {
-    //this needs to do the whole xr tracking volume not just cameras
-    //get the root?
-
-    let tracking_root = match tracking_root_query.get_single() {
-        Ok(e) => e,
-        Err(_) => commands
-            .spawn((SpatialBundle::default(), OpenXRTrackingRoot))
-            .id(),
-    };
-    let right = commands
-        .spawn((XrCameraBundle::new(Eye::Right), OpenXRRightEye))
-        .id();
-    let left = commands
-        .spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye))
-        .id();
-    commands.entity(tracking_root).push_children(&[right, left]);
+    for e in &tracking_root_query {
+        commands.entity(e).despawn_recursive();
+    }
+}
+fn setup_xr_root(
+    mut commands: Commands,
+    tracking_root_query: Query<Entity, With<OpenXRTrackingRoot>>,
+) {
+    if tracking_root_query.get_single().is_err() {
+        info!("Creating XrTrackingRoot!");
+        commands.spawn((SpatialBundle::default(), OpenXRTrackingRoot));
+    }
 }
 
-pub fn action_set_system(action_sets: Res<ActionSets>, session: Res<XrSession>) {
-    let mut active_action_sets = vec![];
-    for i in &action_sets.0 {
-        active_action_sets.push(openxr::ActiveActionSet::new(i));
-    }
-    //info!("action sets: {:#?}", action_sets.0.len());
-    match session.sync_actions(&active_action_sets) {
-        Err(err) => {
-            warn!("{}", err);
-        }
-        _ => {}
-    }
-}
+// pub fn action_set_system(action_sets: Res<ActionSets>, session: Res<XrSession>) {
+//     let mut active_action_sets = vec![];
+//     for i in &action_sets.0 {
+//         active_action_sets.push(openxr::ActiveActionSet::new(i));
+//     }
+//     //info!("action sets: {:#?}", action_sets.0.len());
+//     if let Err(err) = session.sync_actions(&active_action_sets) {
+//         warn!("{}", err);
+//     }
+// }
 
 pub trait Vec2Conv {
     fn to_vec2(&self) -> Vec2;

src/xr_input/oculus_touch.rs

@@ -371,7 +371,7 @@ pub struct OculusController {
     pub aim_space: Option<Handed<Space>>,
 }
 impl OculusController {
-    pub fn new(mut action_sets: ResMut<SetupActionSets>) -> anyhow::Result<Self> {
+    pub fn new(mut action_sets: ResMut<SetupActionSets>) -> eyre::Result<Self> {
         let action_set =
             action_sets.add_action_set("oculus_input", "Oculus Touch Controller Input".into(), 0);
         action_set.new_action(

src/xr_input/prototype_locomotion.rs

@@ -59,28 +59,23 @@ impl Default for PrototypeLocomotionConfig {
 
 pub fn proto_locomotion(
     time: Res<Time>,
-    mut tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
+    mut tracking_root_query: Query<&mut Transform, With<OpenXRTrackingRoot>>,
     oculus_controller: Res<OculusController>,
     frame_state: Res<XrFrameState>,
     xr_input: Res<XrInput>,
-    instance: Res<XrInstance>,
     session: Res<XrSession>,
     views: ResMut<XrViews>,
     mut gizmos: Gizmos,
     config_option: Option<ResMut<PrototypeLocomotionConfig>>,
     action_sets: Res<XrActionSets>,
 ) {
-    match config_option {
-        Some(_) => (),
+    let mut config = match config_option {
+        Some(c) => c,
         None => {
             info!("no locomotion config");
             return;
         }
-    }
-    //i hate this but im too tired to think
-    let mut config = config_option.unwrap();
-    //lock frame
-    let frame_state = *frame_state.lock().unwrap();
+    };
     //get controller
     let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
     let root = tracking_root_query.get_single_mut();
@@ -88,12 +83,11 @@ pub fn proto_locomotion(
         Ok(mut position) => {
             //get the stick input and do some maths
             let stick = controller.thumbstick(Hand::Left);
-            let input = stick.x * position.0.right() + stick.y * position.0.forward();
+            let input = stick.x * *position.right() + stick.y * *position.forward();
             let reference_quat;
             match config.locomotion_type {
                 LocomotionType::Head => {
-                    let v = views.lock().unwrap();
-                    let views = v.get(0);
+                    let views = views.first();
                     match views {
                         Some(view) => {
                             reference_quat = view.pose.orientation.to_quat();
@@ -107,10 +101,9 @@ pub fn proto_locomotion(
                 }
             }
             let (yaw, _pitch, _roll) = reference_quat.to_euler(EulerRot::YXZ);
-            let reference_quat = Quat::from_axis_angle(position.0.up(), yaw);
+            let reference_quat = Quat::from_axis_angle(*position.up(), yaw);
             let locomotion_vec = reference_quat.mul_vec3(input);
-            position.0.translation +=
-                locomotion_vec * config.locomotion_speed * time.delta_seconds();
+            position.translation += locomotion_vec * config.locomotion_speed * time.delta_seconds();
 
             //now time for rotation
 
@@ -123,20 +116,19 @@ pub fn proto_locomotion(
                         return;
                     }
                     let smoth_rot = Quat::from_axis_angle(
-                        position.0.up(),
+                        *position.up(),
                         rot_input * config.smooth_rotation_speed * time.delta_seconds(),
                     );
                     //apply rotation
-                    let v = views.lock().unwrap();
-                    let views = v.get(0);
+                    let views = views.first();
                     match views {
                         Some(view) => {
                             let mut hmd_translation = view.pose.position.to_vec3();
                             hmd_translation.y = 0.0;
-                            let local = position.0.translation;
-                            let global = position.0.rotation.mul_vec3(hmd_translation) + local;
-                            gizmos.circle(global, position.0.up(), 0.1, Color::GREEN);
-                            position.0.rotate_around(global, smoth_rot);
+                            let local = position.translation;
+                            let global = position.rotation.mul_vec3(hmd_translation) + local;
+                            gizmos.circle(global, position.up(), 0.1, Color::GREEN);
+                            position.rotate_around(global, smoth_rot);
                         }
                         None => return,
                     }
@@ -157,18 +149,18 @@ pub fn proto_locomotion(
                             false => -1.0,
                         };
                         let smoth_rot =
-                            Quat::from_axis_angle(position.0.up(), config.snap_angle * dir);
+                            Quat::from_axis_angle(*position.up(), config.snap_angle * dir);
                         //apply rotation
-                        let v = views.lock().unwrap();
-                        let views = v.get(0);
+                        let v = views;
+                        let views = v.first();
                         match views {
                             Some(view) => {
                                 let mut hmd_translation = view.pose.position.to_vec3();
                                 hmd_translation.y = 0.0;
-                                let local = position.0.translation;
-                                let global = position.0.rotation.mul_vec3(hmd_translation) + local;
-                                gizmos.circle(global, position.0.up(), 0.1, Color::GREEN);
-                                position.0.rotate_around(global, smoth_rot);
+                                let local = position.translation;
+                                let global = position.rotation.mul_vec3(hmd_translation) + local;
+                                gizmos.circle(global, position.up(), 0.1, Color::GREEN);
+                                position.rotate_around(global, smoth_rot);
                             }
                             None => return,
                         }

src/xr_input/trackers.rs

@@ -1,4 +1,6 @@
+use bevy::hierarchy::Parent;
 use bevy::log::{debug, info};
+use bevy::math::Quat;
 use bevy::prelude::{
     Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With, Without,
 };
@@ -30,44 +32,45 @@ pub struct OpenXRController;
 pub struct AimPose(pub Transform);
 
 pub fn adopt_open_xr_trackers(
-    query: Query<Entity, Added<OpenXRTracker>>,
+    query: Query<Entity, (With<OpenXRTracker>, Without<Parent>)>,
     mut commands: Commands,
-    tracking_root_query: Query<(Entity, With<OpenXRTrackingRoot>)>,
+    tracking_root_query: Query<Entity, With<OpenXRTrackingRoot>>,
 ) {
     let root = tracking_root_query.get_single();
     match root {
-        Ok(thing) => {
+        Ok(root) => {
             // info!("root is");
             for tracker in query.iter() {
                 info!("we got a new tracker");
-                commands.entity(thing.0).add_child(tracker);
+                commands.entity(root).add_child(tracker);
             }
         }
         Err(_) => info!("root isnt spawned yet?"),
     }
 }
 
+pub fn verify_quat(mut quat: Quat) -> Quat {
+    if quat.length() == 0.0 || !quat.is_finite() {
+        quat = Quat::IDENTITY;
+    }
+    quat.normalize()
+}
+
 pub fn update_open_xr_controllers(
     oculus_controller: Res<OculusController>,
-    mut left_controller_query: Query<(
-        &mut Transform,
-        Option<&mut AimPose>,
-        With<OpenXRLeftController>,
-        Without<OpenXRRightController>,
-    )>,
-    mut right_controller_query: Query<(
-        &mut Transform,
-        Option<&mut AimPose>,
-        With<OpenXRRightController>,
-        Without<OpenXRLeftController>,
-    )>,
+    mut left_controller_query: Query<
+        (&mut Transform, Option<&mut AimPose>),
+        (With<OpenXRLeftController>, Without<OpenXRRightController>),
+    >,
+    mut right_controller_query: Query<
+        (&mut Transform, Option<&mut AimPose>),
+        (With<OpenXRRightController>, Without<OpenXRLeftController>),
+    >,
     frame_state: Res<XrFrameState>,
     xr_input: Res<XrInput>,
     session: Res<XrSession>,
     action_sets: Res<XrActionSets>,
 ) {
-    //lock dat frame?
-    let frame_state = *frame_state.lock().unwrap();
     //get controller
     let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
     //get left controller
@@ -82,7 +85,7 @@ pub fn update_open_xr_controllers(
             Some(mut pose) => {
                 *pose = AimPose(Transform {
                     translation: left_aim_space.0.pose.position.to_vec3(),
-                    rotation: left_aim_space.0.pose.orientation.to_quat(),
+                    rotation: verify_quat(left_aim_space.0.pose.orientation.to_quat()),
                     scale: Vec3::splat(1.0),
                 });
             }
@@ -100,7 +103,7 @@ pub fn update_open_xr_controllers(
     let left_rotataion = left_controller_query.get_single_mut();
     match left_rotataion {
         Ok(mut left_entity) => {
-            left_entity.0.rotation = left_grip_space.0.pose.orientation.to_quat()
+            left_entity.0.rotation = verify_quat(left_grip_space.0.pose.orientation.to_quat())
         }
         Err(_) => (),
     }
@@ -115,7 +118,7 @@ pub fn update_open_xr_controllers(
             Some(mut pose) => {
                 *pose = AimPose(Transform {
                     translation: right_aim_space.0.pose.position.to_vec3(),
-                    rotation: right_aim_space.0.pose.orientation.to_quat(),
+                    rotation: verify_quat(right_aim_space.0.pose.orientation.to_quat()),
                     scale: Vec3::splat(1.0),
                 });
             }
@@ -133,7 +136,7 @@ pub fn update_open_xr_controllers(
     let right_rotataion = right_controller_query.get_single_mut();
     match right_rotataion {
         Ok(mut right_entity) => {
-            right_entity.0.rotation = right_grip_space.0.pose.orientation.to_quat()
+            right_entity.0.rotation = verify_quat(right_grip_space.0.pose.orientation.to_quat())
         }
         Err(_) => (),
     }

src/xr_input/xr_camera.rs

@@ -1,12 +1,94 @@
+use crate::prelude::XrSystems;
+use crate::xr_init::{xr_only, XrCleanup, XrSetup};
 use crate::xr_input::{QuatConv, Vec3Conv};
-use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
+use crate::{locate_views, xr_wait_frame, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
+use bevy::core_pipeline::core_3d::graph::Core3d;
 use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping};
+use bevy::ecs::system::lifetimeless::Read;
 use bevy::math::Vec3A;
 use bevy::prelude::*;
-use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget};
+use bevy::render::camera::{
+    CameraMainTextureUsages, CameraProjection, CameraProjectionPlugin, CameraRenderGraph,
+    RenderTarget,
+};
+use bevy::render::extract_component::{ExtractComponent, ExtractComponentPlugin};
 use bevy::render::primitives::Frustum;
-use bevy::render::view::{ColorGrading, VisibleEntities};
+use bevy::render::view::{
+    update_frusta, ColorGrading, ExtractedView, VisibilitySystems, VisibleEntities,
+};
+use bevy::render::{Render, RenderApp, RenderSet};
+use bevy::transform::TransformSystem;
 use openxr::Fovf;
+use wgpu::TextureUsages;
+
+use super::trackers::{OpenXRLeftEye, OpenXRRightEye, OpenXRTracker, OpenXRTrackingRoot};
+
+pub struct XrCameraPlugin;
+
+impl Plugin for XrCameraPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(CameraProjectionPlugin::<XRProjection>::default());
+        app.add_systems(
+            PreUpdate,
+            xr_camera_head_sync
+                .run_if(xr_only())
+                .after(xr_wait_frame)
+                .after(locate_views),
+        );
+        // a little late latching
+        app.add_systems(
+            PostUpdate,
+            xr_camera_head_sync
+                .before(TransformSystem::TransformPropagate)
+                .run_if(xr_only()),
+        );
+        app.add_systems(
+            PostUpdate,
+            update_frusta::<XRProjection>
+                .after(TransformSystem::TransformPropagate)
+                .before(VisibilitySystems::UpdatePerspectiveFrusta),
+        );
+        app.add_systems(
+            PostUpdate,
+            update_root_transform_components
+                .after(TransformSystem::TransformPropagate)
+                .xr_only(),
+        );
+        app.add_systems(XrSetup, setup_xr_cameras);
+        app.add_systems(XrCleanup, cleanup_xr_cameras);
+        app.add_plugins(ExtractComponentPlugin::<XrCamera>::default());
+        app.add_plugins(ExtractComponentPlugin::<XRProjection>::default());
+        app.add_plugins(ExtractComponentPlugin::<RootTransform>::default());
+        // app.add_plugins(ExtractComponentPlugin::<TransformExtract>::default());
+        // app.add_plugins(ExtractComponentPlugin::<GlobalTransformExtract>::default());
+        let render_app = app.sub_app_mut(RenderApp);
+        render_app.add_systems(
+            Render,
+            (locate_views, xr_camera_head_sync_render_world)
+                .chain()
+                .run_if(xr_only())
+                .in_set(RenderSet::PrepareAssets),
+            // .after(xr_wait_frame)
+            // .after(locate_views),
+        );
+    }
+}
+
+// might be unnesesary since it should be parented to the root
+fn cleanup_xr_cameras(mut commands: Commands, entities: Query<Entity, With<XrCamera>>) {
+    for e in &entities {
+        commands.entity(e).despawn_recursive();
+    }
+}
+
+fn setup_xr_cameras(mut commands: Commands) {
+    commands.spawn((
+        XrCameraBundle::new(Eye::Right),
+        OpenXRRightEye,
+        OpenXRTracker,
+    ));
+    commands.spawn((XrCameraBundle::new(Eye::Left), OpenXRLeftEye, OpenXRTracker));
+}
 
 #[derive(Bundle)]
 pub struct XrCamerasBundle {
@@ -40,13 +122,61 @@ pub struct XrCameraBundle {
     pub tonemapping: Tonemapping,
     pub dither: DebandDither,
     pub color_grading: ColorGrading,
-    pub xr_camera_type: XrCameraType,
+    pub main_texture_usages: CameraMainTextureUsages,
+    pub xr_camera_type: XrCamera,
+    pub root_transform: RootTransform,
 }
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)]
-pub enum XrCameraType {
-    Xr(Eye),
-    Flatscreen,
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component, ExtractComponent)]
+pub struct XrCamera(Eye);
+
+#[derive(Component, ExtractComponent, Clone, Copy, Debug, Default, Deref, DerefMut)]
+pub struct RootTransform(pub GlobalTransform);
+
+fn update_root_transform_components(
+    mut component_query: Query<&mut RootTransform>,
+    root_query: Query<&GlobalTransform, With<OpenXRTrackingRoot>>,
+) {
+    let root = match root_query.get_single() {
+        Ok(v) => v,
+        Err(err) => {
+            warn!("No or too many XrTracking Roots: {}", err);
+            return;
         }
+    };
+    component_query
+        .par_iter_mut()
+        .for_each(|mut root_transform| **root_transform = *root);
+}
+
+// #[derive(Component)]
+// pub(super) struct TransformExtract;
+//
+// impl ExtractComponent for TransformExtract {
+//     type Query = Read<Transform>;
+//
+//     type Filter = ();
+//
+//     type Out = Transform;
+//
+//     fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option<Self::Out> {
+//         Some(*item)
+//     }
+// }
+//
+// #[derive(Component)]
+// pub(super) struct GlobalTransformExtract;
+//
+// impl ExtractComponent for GlobalTransformExtract {
+//     type Query = Read<GlobalTransform>;
+//
+//     type Filter = ();
+//
+//     type Out = GlobalTransform;
+//
+//     fn extract_component(item: bevy::ecs::query::QueryItem<'_, Self::Query>) -> Option<Self::Out> {
+//         Some(*item)
+//     }
+// }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
 pub enum Eye {
@@ -66,7 +196,7 @@ impl XrCameraBundle {
                 viewport: None,
                 ..default()
             },
-            camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME),
+            camera_render_graph: CameraRenderGraph::new(Core3d),
             xr_projection: Default::default(),
             visible_entities: Default::default(),
             frustum: Default::default(),
@@ -76,12 +206,18 @@ impl XrCameraBundle {
             tonemapping: Default::default(),
             dither: DebandDither::Enabled,
             color_grading: Default::default(),
-            xr_camera_type: XrCameraType::Xr(eye),
+            xr_camera_type: XrCamera(eye),
+            main_texture_usages: CameraMainTextureUsages(
+                TextureUsages::RENDER_ATTACHMENT
+                    | TextureUsages::TEXTURE_BINDING
+                    | TextureUsages::COPY_SRC,
+            ),
+            root_transform: default(),
         }
     }
 }
 
-#[derive(Debug, Clone, Component, Reflect)]
+#[derive(Debug, Clone, Component, Reflect, ExtractComponent)]
 #[reflect(Component, Default)]
 pub struct XRProjection {
     pub near: f32,
@@ -240,23 +376,35 @@ impl CameraProjection for XRProjection {
 }
 
 pub fn xr_camera_head_sync(
-    views: ResMut<crate::resources::XrViews>,
-    mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>,
+    views: Res<crate::resources::XrViews>,
+    mut query: Query<(&mut Transform, &XrCamera, &mut XRProjection)>,
 ) {
-    let mut f = || -> Option<()> {
     //TODO calculate HMD position
     for (mut transform, camera_type, mut xr_projection) in query.iter_mut() {
-            let view_idx = match camera_type {
-                XrCameraType::Xr(eye) => *eye as usize,
-                XrCameraType::Flatscreen => return None,
+        let view_idx = camera_type.0 as usize;
+        let view = match views.get(view_idx) {
+            Some(views) => views,
+            None => continue,
         };
-            let v = views.lock().unwrap();
-            let view = v.get(view_idx)?;
         xr_projection.fov = view.fov;
         transform.rotation = view.pose.orientation.to_quat();
         transform.translation = view.pose.position.to_vec3();
     }
-        Some(())
-    };
-    let _ = f();
+}
+
+pub fn xr_camera_head_sync_render_world(
+    views: Res<crate::resources::XrViews>,
+    mut query: Query<(&mut ExtractedView, &XrCamera, &RootTransform)>,
+) {
+    for (mut extracted_view, camera_type, root) in query.iter_mut() {
+        let view_idx = camera_type.0 as usize;
+        let view = match views.get(view_idx) {
+            Some(views) => views,
+            None => continue,
+        };
+        let mut transform = Transform::IDENTITY;
+        transform.rotation = view.pose.orientation.to_quat();
+        transform.translation = view.pose.position.to_vec3();
+        extracted_view.transform = root.mul_transform(transform);
+    }
 }