Merge pull request #3 from awtterpip/new_resource_init

New resource init

Cargo.toml

@@ -9,14 +9,19 @@ edition = "2021"
 anyhow = "1.0.75"
 ash = "0.37.3"
 bevy = { git = "https://github.com/awtterpip/bevy", default-features = false, features = ["bevy_render"] }
-openxr = { version = "0.17.1" }
+openxr = { version = "0.17.1", features = ["mint"] }
+mint = "0.5.9"
 wgpu = "0.16.0"
 wgpu-core = { version = "0.16.0", features = ["vulkan"] }
 wgpu-hal = "0.16.0"
 
 [dev-dependencies]
 bevy = { git = "https://github.com/awtterpip/bevy" }
+color-eyre = "0.6.2"
 
 [[example]]
 name = "xr"
 path = "examples/xr.rs"
+
+[profile.release]
+debug = true

examples/xr.rs

@@ -1,27 +1,44 @@
-//! A simple 3D scene with light shining over a cube sitting on a plane.
+use bevy::core_pipeline::core_3d;
-use bevy_openxr::{DefaultXrPlugins, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
+use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping};
+use bevy::ecs::prelude::{Bundle, Component, ReflectComponent};
+
+use bevy::math::Mat4;
+use bevy::prelude::Camera3d;
+use bevy::reflect::{std_traits::ReflectDefault, Reflect};
+use bevy::render::view::ColorGrading;
+use bevy::render::{
+    camera::{Camera, CameraProjection, CameraRenderGraph},
+    primitives::Frustum,
+    view::VisibleEntities,
+};
+use bevy::transform::components::{GlobalTransform, Transform};
+//  mostly copied from https://github.com/blaind/bevy_openxr/tree/main/crates/bevy_openxr/src/render_graph/camera
+use openxr::Fovf;
+
+use bevy::render::camera::CameraProjectionPlugin;
+use bevy::render::view::{update_frusta, VisibilitySystems};
+use bevy::transform::TransformSystem;
 use bevy::{prelude::*, render::camera::RenderTarget};
-use bevy::prelude::Component;
-use bevy::render::camera::Viewport;
 use bevy_openxr::input::XrInput;
-use bevy_openxr::resources::{XrInstance, XrSession, XrViews};
+use bevy_openxr::resources::{XrFrameState, XrSession, XrViews};
+use bevy_openxr::xr_input::controllers::XrControllerType;
+use bevy_openxr::xr_input::oculus_touch::OculusController;
+use bevy_openxr::xr_input::{OpenXrInput, QuatConv, Vec3Conv};
+use bevy_openxr::{DefaultXrPlugins, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
+use openxr::ActiveActionSet;
 
 fn main() {
+    color_eyre::install().unwrap();
+
+    info!("Running `openxr-6dof` skill");
     App::new()
         .add_plugins(DefaultXrPlugins)
+        .add_plugins(OpenXrInput::new(XrControllerType::OculusTouch))
         .add_systems(Startup, setup)
-        .add_systems(Update, head_movement)
+        .add_systems(Update, hands)
         .run();
 }
 
-#[derive(Component)]
-enum CameraType {
-    Left,
-    Right,
-    Middle,
-}
-
-
 /// set up a simple 3D scene
 fn setup(
     mut commands: Commands,
@@ -36,7 +53,7 @@ fn setup(
     });
     // cube
     commands.spawn(PbrBundle {
-        mesh: meshes.add(Mesh::from(shape::Cube { size: 1.0 })),
+        mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
         material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
         transform: Transform::from_xyz(0.0, 0.5, 0.0),
         ..default()
@@ -55,116 +72,37 @@ fn setup(
     commands.spawn((Camera3dBundle {
         transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y),
         ..default()
-    }, CameraType::Middle));
+    },));
-
-    // let viewport = Viewport{
-    //     physical_position: Default::default(),
-    //     physical_size: UVec2::splat(2000),
-    //     depth: 0.0..1.0,
-    // };
-
-    commands.spawn((Camera3dBundle {
-        transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y),
-        camera: Camera {
-            order: -1,
-            target: RenderTarget::TextureView(LEFT_XR_TEXTURE_HANDLE),
-            viewport: None,
-            ..default()
-        },
-        ..default()
-    }, CameraType::Left));
-    commands.spawn((Camera3dBundle {
-        transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y),
-        camera: Camera {
-            order: -1,
-            target: RenderTarget::TextureView(RIGHT_XR_TEXTURE_HANDLE),
-            viewport: None,
-            ..default()
-        },
-        ..default()
-    }, CameraType::Right));
-}
-fn head_movement(views: ResMut<XrViews>, mut query: Query<(&mut Transform, &Camera, &CameraType)>) {
-    let views = views.lock().unwrap();
-    let mut f = || -> Option<()> {
-        let midpoint = (views.get(0)?.pose.position.to_vec3()
-            + views.get(1)?.pose.position.to_vec3())
-            / 2.;
-        for (mut t, _, camera_type) in query.iter_mut() {
-            match camera_type {
-                CameraType::Left => {
-                    t.translation = views.get(0)?.pose.position.to_vec3()
-                },
-                CameraType::Right => {
-                    t.translation = views.get(1)?.pose.position.to_vec3()
-                },
-                CameraType::Middle => {
-                    t.translation = midpoint;
-                },
-            }
-        }
-        let left_rot = views.get(0).unwrap().pose.orientation.to_quat();
-        let right_rot = views.get(1).unwrap().pose.orientation.to_quat();
-        let mid_rot = if left_rot.dot(right_rot) >= 0. {
-            left_rot.slerp(right_rot, 0.5)
-        } else {
-            right_rot.slerp(left_rot, 0.5)
-        };
-        for (mut t, _, camera_type) in query.iter_mut() {
-            match camera_type {
-                CameraType::Left => {
-                    t.rotation = left_rot
-                },
-                CameraType::Right => {
-                    t.rotation = right_rot
-                },
-                CameraType::Middle => {
-                    t.rotation = mid_rot;
-                },
-            }
-        }
-
-
-    //     for (mut projection, mut transform, eye) in cam.iter_mut() {
-    //     let view_idx = match eye {
-    //         Eye::Left => 0,
-    //         Eye::Right => 1,
-    //     };
-    //     let view = views.get(view_idx).unwrap();
-    //
-    //     projection.fov = view.fov;
-    //
-    //     transform.rotation = view.pose.orientation.to_quat();
-    //     let pos = view.pose.position;
-    //     transform.translation = pos.to_vec3();
-    // }
-
-        Some(())
-    };
-    f();
-}
-pub trait Vec3Conv {
-    fn to_vec3(&self) -> Vec3;
 }
 
-impl Vec3Conv for openxr::Vector3f {
+fn hands(
-    fn to_vec3(&self) -> Vec3 {
+    mut gizmos: Gizmos,
-        Vec3::new(self.x, self.y, self.z)
+    oculus_controller: Res<OculusController>,
-    }
+    frame_state: Res<XrFrameState>,
-}
+    xr_input: Res<XrInput>,
-pub trait QuatConv {
+) {
-    fn to_quat(&self) -> Quat;
+    let frame_state = *frame_state.lock().unwrap();
-}
 
-impl QuatConv for openxr::Quaternionf {
+        let right_controller = oculus_controller
-    fn to_quat(&self) -> Quat {
+            .grip_space
-        Quat::from_xyzw(self.x, self.y, self.z, self.w)
+            .right
-    }
+            .relate(&**&xr_input.stage, frame_state.predicted_display_time)
+            .unwrap();
+        let left_controller = oculus_controller
+            .grip_space
+            .left
+            .relate(&**&xr_input.stage, frame_state.predicted_display_time)
+            .unwrap();
+        gizmos.rect(
+            right_controller.0.pose.position.to_vec3(),
+            right_controller.0.pose.orientation.to_quat(),
+            Vec2::new(0.05, 0.2),
+            Color::YELLOW_GREEN,
+        );
+        gizmos.rect(
+            left_controller.0.pose.position.to_vec3(),
+            left_controller.0.pose.orientation.to_quat(),
+            Vec2::new(0.05, 0.2),
+            Color::YELLOW_GREEN,
+        );
 }
-
-// fn head_movement(right_camera: Query<(&mut Transform, &RightCamera), Without<LeftCamera>>, left_camera: Query<(&mut Transform, &LeftCamera), Without<RightCamera>>, xr_input: Res<bevy_openxr::input::XrInput>, instance: Res<XrInstance>, session: Res<XrSession>) {
-//
-//     // let stage =
-//     //     session.create_reference_space(openxr::ReferenceSpaceType::VIEW, openxr::Posef::IDENTITY).unwrap();
-//     // eprintln!("a: {:#?}", stage.locate(&xr_input.stage, xr_input.action_set.).unwrap().pose);
-// }

src/graphics/mod.rs

@@ -1,12 +1,34 @@
 mod vulkan;
 
-use bevy::render::renderer::{RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter};
+use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue};
 use bevy::window::RawHandleWrapper;
 use wgpu::Instance;
 
 use crate::input::XrInput;
-use crate::resources::{XrInstance, XrSession, XrEnvironmentBlendMode, XrSessionRunning, XrFrameWaiter, XrSwapchain, XrViews, XrFrameState};
+use crate::resources::{
+    XrEnvironmentBlendMode, XrFrameState, XrFrameWaiter, XrInstance, XrSession, XrSessionRunning,
+    XrSwapchain, XrViews, XrResolution, XrFormat,
+};
 
-pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Result<(RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter, Instance, XrInstance, XrSession, XrEnvironmentBlendMode, XrSessionRunning, XrFrameWaiter, XrSwapchain, XrInput, XrViews, XrFrameState)>{
+pub fn initialize_xr_graphics(
+    window: Option<RawHandleWrapper>,
+) -> anyhow::Result<(
+    RenderDevice,
+    RenderQueue,
+    RenderAdapterInfo,
+    RenderAdapter,
+    Instance,
+    XrInstance,
+    XrSession,
+    XrEnvironmentBlendMode,
+    XrResolution,
+    XrFormat,
+    XrSessionRunning,
+    XrFrameWaiter,
+    XrSwapchain,
+    XrInput,
+    XrViews,
+    XrFrameState,
+)> {
     vulkan::initialize_xr_graphics(window)
 }

src/graphics/vulkan.rs

@@ -1,6 +1,6 @@
 use std::ffi::{c_void, CString};
-use std::sync::{Arc, Mutex};
 use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, Mutex};
 
 use anyhow::Context;
 use ash::vk::{self, Handle};
@@ -13,11 +13,14 @@ use wgpu::{Instance, Texture};
 
 use crate::input::XrInput;
 use crate::resources::{
-    XrEnvironmentBlendMode, XrFrameWaiter, XrInstance, XrSession, XrSessionRunning, XrSwapchain, Swapchain, SwapchainInner, XrViews, XrFrameState,
+    Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFrameState, XrFrameWaiter, XrInstance,
+    XrSession, XrSessionRunning, XrSwapchain, XrViews, XrResolution, XrFormat,
 };
 use crate::VIEW_TYPE;
 
-pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Result<(
+pub fn initialize_xr_graphics(
+    window: Option<RawHandleWrapper>,
+) -> anyhow::Result<(
     RenderDevice,
     RenderQueue,
     RenderAdapterInfo,
@@ -26,6 +29,8 @@ pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Resul
     XrInstance,
     XrSession,
     XrEnvironmentBlendMode,
+    XrResolution,
+    XrFormat,
     XrSessionRunning,
     XrFrameWaiter,
     XrSwapchain,
@@ -281,7 +286,10 @@ pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Resul
         .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0])
         .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb);
 
-    let resolution = uvec2(views[0].recommended_image_rect_width, views[0].recommended_image_rect_height);
+    let resolution = uvec2(
+        views[0].recommended_image_rect_width,
+        views[0].recommended_image_rect_height,
+    );
 
     let handle = session
         .create_swapchain(&xr::SwapchainCreateInfo {
@@ -361,19 +369,24 @@ pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Resul
         xr_instance.clone().into(),
         session.clone().into_any_graphics().into(),
         blend_mode.into(),
+        resolution.into(),
+        swapchain_format.into(),
         AtomicBool::new(false).into(),
         Mutex::new(frame_wait).into(),
-        Mutex::new(Swapchain::Vulkan(SwapchainInner {
+        Swapchain::Vulkan(SwapchainInner {
-            stream: frame_stream,
+            stream: Mutex::new(frame_stream),
-            handle,
+            handle: Mutex::new(handle),
-            resolution,
-            format: swapchain_format,
             buffers,
-            image_index: 0,
+            image_index: Mutex::new(0),
-        })).into(),
+        })
+        .into(),
         XrInput::new(xr_instance, session.into_any_graphics())?,
         Mutex::default().into(),
-        Mutex::default().into(),
+        Mutex::new(xr::FrameState {
+            predicted_display_time: xr::Time::from_nanos(1),
+            predicted_display_period: xr::Duration::from_nanos(1),
+            should_render: true,
+        }).into(),
     ))
 }
 

src/input.rs

@@ -7,57 +7,58 @@ type XrPose = (Vec3, Quat);
 
 #[derive(Clone, Resource)]
 pub struct XrInput {
-    pub action_set: xr::ActionSet,
+    //pub action_set: xr::ActionSet,
-    pub hand_pose: xr::Action<xr::Posef>,
+    //pub hand_pose: xr::Action<xr::Posef>,
-    pub right_space: Arc<xr::Space>,
+    //pub right_space: Arc<xr::Space>,
-    pub left_space: Arc<xr::Space>,
+    //pub left_space: Arc<xr::Space>,
     pub stage: Arc<xr::Space>,
 }
 
 impl XrInput {
-    pub fn new(
+    pub fn new(instance: xr::Instance, session: xr::Session<xr::AnyGraphics>) -> xr::Result<Self> {
-        instance: xr::Instance,
+        // let action_set = instance.create_action_set("input", "input pose information", 0)?;
-        session: xr::Session<xr::AnyGraphics>,
+        // let left_hand_subaction_path = instance.string_to_path("/user/hand/left").unwrap();
-    ) -> xr::Result<Self> {
+        // let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap();
-        let action_set = instance.create_action_set("input", "input pose information", 0)?;
+        // let left_hand_grip_pose_path = instance
-        let left_hand_subaction_path = instance.string_to_path("/user/hand/left").unwrap();
+        //     .string_to_path("/user/hand/left/input/grip/pose")
-        let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap();
+        //     .unwrap();
-        let left_hand_grip_pose_path = instance
+        // let right_hand_grip_pose_path = instance
-            .string_to_path("/user/hand/left/input/grip/pose")
+        //     .string_to_path("/user/hand/right/input/grip/pose")
-            .unwrap();
+        //     .unwrap();
-        let right_hand_grip_pose_path = instance
+        // let hand_pose = action_set.create_action::<xr::Posef>(
-            .string_to_path("/user/hand/right/input/grip/pose")
+        //     "hand_pose",
-            .unwrap();
+        //     "Hand Pose",
-        let hand_pose =
+        //     &[left_hand_subaction_path, right_hand_subaction_path],
-            action_set.create_action::<xr::Posef>("hand_pose", "Hand Pose", &[left_hand_subaction_path, right_hand_subaction_path])?;
+        // )?;
-       /* let left_action =
+        // /* let left_action =
-            action_set.create_action::<xr::Posef>("left_hand", "Left Hand Controller", &[])?;*/
+        // action_set.create_action::<xr::Posef>("left_hand", "Left Hand Controller", &[])?;*/
-        instance.suggest_interaction_profile_bindings(
+        // instance.suggest_interaction_profile_bindings(
-            instance.string_to_path("/interaction_profiles/khr/simple_controller")?,
+        //     instance.string_to_path("/interaction_profiles/khr/simple_controller")?,
-            &[
+        //     &[
-                xr::Binding::new(
+        //         xr::Binding::new(&hand_pose, right_hand_grip_pose_path),
-                    &hand_pose,
+        //         xr::Binding::new(&hand_pose, left_hand_grip_pose_path),
-                    right_hand_grip_pose_path,
+        //     ],
-                ),
+        // )?;
-                xr::Binding::new(
+        //
-                    &hand_pose,
+        // let right_space = hand_pose.create_space(
-                    left_hand_grip_pose_path,
+        //     session.clone(),
-                ),
+        //     right_hand_subaction_path,
-            ],
+        //     xr::Posef::IDENTITY,
-        )?;
+        // )?;
-
+        // let left_space = hand_pose.create_space(
-        let right_space =
+        //     session.clone(),
-            hand_pose.create_space(session.clone(), right_hand_subaction_path, xr::Posef::IDENTITY)?;
+        //     left_hand_subaction_path,
-        let left_space =
+        //     xr::Posef::IDENTITY,
-            hand_pose.create_space(session.clone(), left_hand_subaction_path, xr::Posef::IDENTITY)?;
+        // )?;
         let stage =
             session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?;
-        session.attach_action_sets(&[&action_set])?;
+        //session.attach_action_sets(&[&action_set])?;
+        //session.attach_action_sets(&[])?;
         Ok(Self {
-            action_set,
+            //action_set,
-            hand_pose,
+            //hand_pose,
-            right_space: Arc::new(right_space),
+            // right_space: Arc::new(right_space),
-            left_space: Arc::new(left_space),
+            // left_space: Arc::new(left_space),
             stage: Arc::new(stage),
         })
     }

src/lib.rs

@@ -1,18 +1,27 @@
+mod graphics;
 pub mod input;
 pub mod resource_macros;
 pub mod resources;
-mod graphics;
+pub mod xr_input;
 
 use std::sync::{Arc, Mutex};
 
+use crate::xr_input::oculus_touch::ActionSets;
+use bevy::app::PluginGroupBuilder;
 use bevy::ecs::system::SystemState;
 use bevy::prelude::*;
-use bevy::render::camera::{ManualTextureViews, ManualTextureView, ManualTextureViewHandle};
+use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews};
-use bevy::render::{FutureRendererResources, RenderPlugin, RenderApp, Render, RenderSet};
+use bevy::render::pipelined_rendering::RenderExtractApp;
+use bevy::render::renderer::{
+    render_system, RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue,
+};
+use bevy::render::settings::RenderSettings;
+use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet};
 use bevy::window::{PrimaryWindow, RawHandleWrapper};
 use input::XrInput;
-use resources::*;
 use openxr as xr;
+use resources::*;
+use wgpu::Instance;
 
 const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
 
@@ -31,6 +40,8 @@ pub struct FutureXrResources(
                 XrInstance,
                 XrSession,
                 XrEnvironmentBlendMode,
+                XrResolution,
+                XrFormat,
                 XrSessionRunning,
                 XrFrameWaiter,
                 XrSwapchain,
@@ -44,36 +55,57 @@ pub struct FutureXrResources(
 
 impl Plugin for OpenXrPlugin {
     fn build(&self, app: &mut App) {
-        let future_renderer_resources_wrapper = Arc::new(Mutex::new(None));
-        app.insert_resource(FutureRendererResources(
-            future_renderer_resources_wrapper.clone(),
-        ));
-
         let future_xr_resources_wrapper = Arc::new(Mutex::new(None));
-        app.insert_resource(FutureXrResources(
+        app.insert_resource(FutureXrResources(future_xr_resources_wrapper.clone()));
-            future_xr_resources_wrapper.clone()
-        ));
 
         let mut system_state: SystemState<Query<&RawHandleWrapper, With<PrimaryWindow>>> =
             SystemState::new(&mut app.world);
         let primary_window = system_state.get(&app.world).get_single().ok().cloned();
 
-        bevy::tasks::IoTaskPool::get()
+        let (
-            .spawn_local(async move {
+            device,
-                let (device, queue, adapter_info, render_adapter, instance, xr_instance, session, blend_mode, session_running, frame_waiter, swapchain, input, views, frame_state) = graphics::initialize_xr_graphics(primary_window).unwrap();
+            queue,
+            adapter_info,
+            render_adapter,
+            instance,
+            xr_instance,
+            session,
+            blend_mode,
+            resolution,
+            format,
+            session_running,
+            frame_waiter,
+            swapchain,
+            input,
+            views,
+            frame_state,
+        ) = graphics::initialize_xr_graphics(primary_window).unwrap();
         debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
         debug!("Configured wgpu adapter Features: {:#?}", device.features());
-                let mut future_renderer_resources_inner =
-                    future_renderer_resources_wrapper.lock().unwrap();
-                *future_renderer_resources_inner =
-                    Some((device, queue, adapter_info, render_adapter, instance));
         let mut future_xr_resources_inner = future_xr_resources_wrapper.lock().unwrap();
-                *future_xr_resources_inner =
+        *future_xr_resources_inner = Some((
-                    Some((xr_instance, session, blend_mode, session_running, frame_waiter, swapchain, input, views, frame_state));
+            xr_instance,
-            })
+            session,
-            .detach();
+            blend_mode,
-
+            resolution,
-        
+            format,
+            session_running,
+            frame_waiter,
+            swapchain,
+            input,
+            views,
+            frame_state,
+        ));
+        app.insert_resource(ActionSets(vec![]));
+        app.add_plugins(DefaultPlugins.set(RenderPlugin {
+            render_settings: RenderSettings::Manual(
+                device,
+                queue,
+                adapter_info,
+                render_adapter,
+                Mutex::new(instance),
+            ),
+        }));
     }
 
     fn ready(&self, app: &App) -> bool {
@@ -84,78 +116,117 @@ impl Plugin for OpenXrPlugin {
     }
 
     fn finish(&self, app: &mut App) {
-        if let Some(future_renderer_resources) =
+        if let Some(future_renderer_resources) = app.world.remove_resource::<FutureXrResources>() {
-            app.world.remove_resource::<FutureXrResources>()
+            let (
-        {
+                xr_instance,
-            let (instance, session, blend_mode, session_running, frame_waiter, swapchain, input, views, frame_state) =
+                session,
-                future_renderer_resources.0.lock().unwrap().take().unwrap();
+                blend_mode,
+                resolution,
+                format,
+                session_running,
+                frame_waiter,
+                swapchain,
+                input,
+                views,
+                frame_state,
+            ) = future_renderer_resources.0.lock().unwrap().take().unwrap();
 
-            app.insert_resource(instance.clone())
+            let action_sets = app.world.resource::<ActionSets>().clone();
+
+            app.insert_resource(xr_instance.clone())
                 .insert_resource(session.clone())
                 .insert_resource(blend_mode.clone())
+                .insert_resource(resolution.clone())
+                .insert_resource(format.clone())
                 .insert_resource(session_running.clone())
                 .insert_resource(frame_waiter.clone())
                 .insert_resource(swapchain.clone())
                 .insert_resource(input.clone())
                 .insert_resource(views.clone())
-                .insert_resource(frame_state.clone());
+                .insert_resource(frame_state.clone())
+                .insert_resource(action_sets.clone());
 
-            let swapchain_mut = swapchain.lock().unwrap();
+            let (left, right) = swapchain.get_render_views();
-            let (left, right) = swapchain_mut.get_render_views();
-            let format = swapchain_mut.format();
             let left = ManualTextureView {
                 texture_view: left.into(),
-                size: swapchain_mut.resolution(),
+                size: *resolution,
-                format,
+                format: *format,
             };
             let right = ManualTextureView {
                 texture_view: right.into(),
-                size: swapchain_mut.resolution(),
+                size: *resolution,
-                format,
+                format: *format,
             };
             let mut manual_texture_views = app.world.resource_mut::<ManualTextureViews>();
             manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left);
             manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right);
             drop(manual_texture_views);
-            drop(swapchain_mut);
+            let pipeline_app = app.sub_app_mut(RenderExtractApp);
+            pipeline_app
+                .insert_resource(xr_instance.clone())
+                .insert_resource(session.clone())
+                .insert_resource(blend_mode.clone())
+                .insert_resource(resolution.clone())
+                .insert_resource(format.clone())
+                .insert_resource(session_running.clone())
+                .insert_resource(frame_waiter.clone())
+                .insert_resource(swapchain.clone())
+                .insert_resource(input.clone())
+                .insert_resource(views.clone())
+                .insert_resource(frame_state.clone())
+                .insert_resource(action_sets.clone());
+            drop(pipeline_app);
             let render_app = app.sub_app_mut(RenderApp);
 
-            render_app.insert_resource(instance)
+            render_app
+                .insert_resource(xr_instance)
                 .insert_resource(session)
                 .insert_resource(blend_mode)
+                .insert_resource(resolution)
+                .insert_resource(format)
                 .insert_resource(session_running)
                 .insert_resource(frame_waiter)
                 .insert_resource(swapchain)
                 .insert_resource(input)
                 .insert_resource(views)
-                .insert_resource(frame_state);
+                .insert_resource(frame_state)
+                .insert_resource(action_sets);
 
-            render_app.add_systems(Render, (pre_frame.in_set(RenderSet::Prepare).before(post_frame), post_frame.in_set(RenderSet::Prepare), post_queue_submit.in_set(RenderSet::Cleanup)));
+            render_app.add_systems(
+                Render,
+                (
+                    begin_frame.before(render_system).after(RenderSet::ExtractCommands),
+                    locate_views.before(render_system),
+                    end_frame.after(render_system),
+                ),
+            );
         }
-        
     }
 }
 
 pub struct DefaultXrPlugins;
 
 impl PluginGroup for DefaultXrPlugins {
-    fn build(self) -> bevy::app::PluginGroupBuilder {
+    fn build(self) -> PluginGroupBuilder {
-        DefaultPlugins
+        let mut group = PluginGroupBuilder::start::<Self>();
-            .build()
+        group = group.add(OpenXrPlugin);
-            .add_before::<RenderPlugin, _>(OpenXrPlugin)
+        group
     }
 }
 
-pub fn pre_frame(
+pub fn begin_frame(
     instance: Res<XrInstance>,
     session: Res<XrSession>,
     session_running: Res<XrSessionRunning>,
-    frame_state: Res<XrFrameState>,
+    resolution: Res<XrResolution>,
-    frame_waiter: Res<XrFrameWaiter>,
+    format: Res<XrFormat>,
     swapchain: Res<XrSwapchain>,
-    xr_input: Res<XrInput>,
+    frame_waiter: Res<XrFrameWaiter>,
+    frame_state: Res<XrFrameState>,
     mut manual_texture_views: ResMut<ManualTextureViews>,
-){
+) {
+    {
+        let _span = info_span!("xr_poll_events");
         while let Some(event) = instance.poll_event(&mut Default::default()).unwrap() {
             use xr::Event::*;
             match event {
@@ -172,78 +243,91 @@ pub fn pre_frame(
                             session.end().unwrap();
                             session_running.store(false, std::sync::atomic::Ordering::Relaxed);
                         }
-                    xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
+                        xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => return,
-                        return
-                    }
                         _ => {}
                     }
                 }
-            InstanceLossPending(_) => {
+                InstanceLossPending(_) => return,
-                return
-            }
                 EventsLost(e) => {
                     warn!("lost {} XR events", e.lost_event_count());
                 }
                 _ => {}
             }
         }
-    if !session_running.load(std::sync::atomic::Ordering::Relaxed) {
-        // Don't grind up the CPU
-        std::thread::sleep(std::time::Duration::from_millis(10));
-        return
     }
-
+    {
-    *frame_state.lock().unwrap() = Some(frame_waiter.lock().unwrap().wait().unwrap());
+        let _span = info_span!("xr_wait_frame").entered();
-
+        *frame_state.lock().unwrap() = frame_waiter.lock().unwrap().wait().unwrap();
-    let mut swapchain = swapchain.lock().unwrap();
+    }
-
+    {
-    swapchain.begin().unwrap();
+        let _span = info_span!("xr_begin_frame").entered();
-    swapchain.update_render_views();
+        swapchain.begin().unwrap()
+    }
+    {
+        let _span = info_span!("xr_acquire_image").entered();
+        swapchain.acquire_image().unwrap()
+    }
+    {
+        let _span = info_span!("xr_wait_image").entered();
+        swapchain.wait_image().unwrap();
+    }
+    {
+        let _span = info_span!("xr_update_manual_texture_views").entered();
         let (left, right) = swapchain.get_render_views();
-    let active_action_set = xr::ActiveActionSet::new(&xr_input.action_set);
-    match session.sync_actions(&[active_action_set]) {
-        Err(err) => {
-            eprintln!("{}", err);
-        }
-        _ => {}
-    }
-    let format = swapchain.format();
         let left = ManualTextureView {
             texture_view: left.into(),
-        size: swapchain.resolution(),
+            size: **resolution,
-        format,
+            format: **format,
         };
         let right = ManualTextureView {
             texture_view: right.into(),
-        size: swapchain.resolution(),
+            size: **resolution,
-        format,
+            format: **format,
         };
         manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left);
         manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right);
+    }
 }
 
-pub fn post_frame(
+pub fn end_frame(
+    xr_frame_state: Res<XrFrameState>,
+    views: Res<XrViews>,
+    input: Res<XrInput>,
+    swapchain: Res<XrSwapchain>,
+    resolution: Res<XrResolution>,
+    environment_blend_mode: Res<XrEnvironmentBlendMode>,
+) {
+    {
+        let _span = info_span!("xr_release_image").entered();
+        swapchain.release_image().unwrap();
+    }
+    {
+        let _span = info_span!("xr_end_frame").entered();
+        swapchain
+            .end(
+                xr_frame_state.lock().unwrap().predicted_display_time,
+                &*views.lock().unwrap(),
+                &input.stage,
+                **resolution,
+                **environment_blend_mode,
+            )
+            .unwrap();
+    }
+}
+
+pub fn locate_views(
     views: Res<XrViews>,
     input: Res<XrInput>,
     session: Res<XrSession>,
     xr_frame_state: Res<XrFrameState>,
 ) {
-    *views.lock().unwrap() = session.locate_views(
+    let _span = info_span!("xr_locate_views").entered();
+    *views.lock().unwrap() = session
+        .locate_views(
             VIEW_TYPE,
-        xr_frame_state.lock().unwrap().unwrap().predicted_display_time,
+            xr_frame_state.lock().unwrap().predicted_display_time,
             &input.stage,
-    ).unwrap().1;
+        )
-}
+        .unwrap()
-
+        .1;
-pub fn post_queue_submit(
-    xr_frame_state: Res<XrFrameState>,
-    views: Res<XrViews>,
-    input: Res<XrInput>,
-    swapchain: Res<XrSwapchain>,
-    environment_blend_mode: Res<XrEnvironmentBlendMode>,
-) {
-    let xr_frame_state = xr_frame_state.lock().unwrap().unwrap();
-    let views = &*views.lock().unwrap();
-    let stage = &input.stage;
-    swapchain.lock().unwrap().post_queue_submit(xr_frame_state, views, stage, **environment_blend_mode).unwrap();
 }

src/resource_macros.rs

@@ -23,7 +23,7 @@ macro_rules! xr_resource_wrapper {
                 Self::new(value)
             }
         }
-    }
+    };
 }
 
 #[macro_export]
@@ -51,8 +51,8 @@ macro_rules! xr_arc_resource_wrapper {
                 Self::new(value)
             }
         }
-    }
+    };
 }
 
-pub use xr_resource_wrapper;
 pub use xr_arc_resource_wrapper;
+pub use xr_resource_wrapper;

src/resources.rs

@@ -8,11 +8,12 @@ use openxr as xr;
 xr_resource_wrapper!(XrInstance, xr::Instance);
 xr_resource_wrapper!(XrSession, xr::Session<xr::AnyGraphics>);
 xr_resource_wrapper!(XrEnvironmentBlendMode, xr::EnvironmentBlendMode);
-xr_resource_wrapper!(XrViewConfigurationViews, Vec<xr::ViewConfigurationView>);
+xr_resource_wrapper!(XrResolution, UVec2);
+xr_resource_wrapper!(XrFormat, wgpu::TextureFormat);
 xr_arc_resource_wrapper!(XrSessionRunning, AtomicBool);
 xr_arc_resource_wrapper!(XrFrameWaiter, Mutex<xr::FrameWaiter>);
-xr_arc_resource_wrapper!(XrSwapchain, Mutex<Swapchain>);
+xr_arc_resource_wrapper!(XrSwapchain, Swapchain);
-xr_arc_resource_wrapper!(XrFrameState, Mutex<Option<xr::FrameState>>);
+xr_arc_resource_wrapper!(XrFrameState, Mutex<xr::FrameState>);
 xr_arc_resource_wrapper!(XrViews, Mutex<Vec<xr::View>>);
 
 pub enum Swapchain {
@@ -20,65 +21,70 @@ pub enum Swapchain {
 }
 
 impl Swapchain {
-    pub(crate) fn begin(&mut self) -> xr::Result<()> {
+    pub(crate) fn begin(&self) -> xr::Result<()> {
         match self {
-            Swapchain::Vulkan(swap) => swap.begin(),
+            Swapchain::Vulkan(swapchain) => swapchain.begin(),
-        }
-    }
-
-    pub(crate) fn update_render_views(&mut self) {
-        match self {
-            Swapchain::Vulkan(swap) => swap.update_render_views(),
         }
     }
 
     pub(crate) fn get_render_views(&self) -> (wgpu::TextureView, wgpu::TextureView) {
         match self {
-            Swapchain::Vulkan(swap) => swap.get_render_views(),
+            Swapchain::Vulkan(swapchain) => swapchain.get_render_views(),
         }
     }
 
-    pub(crate) fn format(&self) -> wgpu::TextureFormat {
+    pub(crate) fn acquire_image(&self) -> xr::Result<()> {
         match self {
-            Swapchain::Vulkan(swap) => swap.format
+            Swapchain::Vulkan(swapchain) => swapchain.acquire_image(),
         }
     }
 
-    pub(crate) fn resolution(&self) -> UVec2 {
+    pub(crate) fn wait_image(&self) -> xr::Result<()> {
         match self {
-            Swapchain::Vulkan(swap) => swap.resolution,
+            Swapchain::Vulkan(swapchain) => swapchain.wait_image(),
         }
     }
 
-    pub(crate) fn post_queue_submit(
+    pub(crate) fn release_image(&self) -> xr::Result<()> {
-        &mut self,
+        match self {
-        xr_frame_state: xr::FrameState,
+            Swapchain::Vulkan(swapchain) => swapchain.release_image(),
+        }
+    }
+
+    pub(crate) fn end(
+        &self,
+        predicted_display_time: xr::Time,
         views: &[openxr::View],
         stage: &xr::Space,
+        resolution: UVec2,
         environment_blend_mode: xr::EnvironmentBlendMode,
     ) -> xr::Result<()> {
         match self {
-            Swapchain::Vulkan(swap) => swap.post_queue_submit(xr_frame_state, views, stage, environment_blend_mode),
+            Swapchain::Vulkan(swapchain) => swapchain.end(
+                predicted_display_time,
+                views,
+                stage,
+                resolution,
+                environment_blend_mode,
+            ),
         }
     }
 }
 
 pub struct SwapchainInner<G: xr::Graphics> {
-    pub(crate) stream: xr::FrameStream<G>,
+    pub(crate) stream: Mutex<xr::FrameStream<G>>,
-    pub(crate) handle: xr::Swapchain<G>,
+    pub(crate) handle: Mutex<xr::Swapchain<G>>,
-    pub(crate) resolution: UVec2,
-    pub(crate) format: wgpu::TextureFormat,
     pub(crate) buffers: Vec<wgpu::Texture>,
-    pub(crate) image_index: usize,
+    pub(crate) image_index: Mutex<usize>,
 }
 
 impl<G: xr::Graphics> SwapchainInner<G> {
-    fn begin(&mut self) -> xr::Result<()> {
+    fn begin(&self) -> xr::Result<()> {
-        self.stream.begin()
+        self.stream.lock().unwrap().begin()
     }
 
     fn get_render_views(&self) -> (wgpu::TextureView, wgpu::TextureView) {
-        let texture = &self.buffers[self.image_index];
+        let texture = &self.buffers[*self.image_index.lock().unwrap()];
 
         (
             texture.create_view(&wgpu::TextureViewDescriptor {
@@ -95,30 +101,41 @@ impl<G: xr::Graphics> SwapchainInner<G> {
         )
     }
 
-    fn update_render_views(&mut self) {
+    fn acquire_image(&self) -> xr::Result<()> {
-        let image_index = self.handle.acquire_image().unwrap();
+        let image_index = self.handle.lock().unwrap().acquire_image()?;
-        self.handle.wait_image(xr::Duration::INFINITE).unwrap();
+        *self.image_index.lock().unwrap() = image_index as _;
-
+        Ok(())
-        self.image_index = image_index as _;
     }
 
-    fn post_queue_submit(
+    fn wait_image(&self) -> xr::Result<()> {
-        &mut self,
+        self.handle
-        xr_frame_state: xr::FrameState,
+            .lock()
+            .unwrap()
+            .wait_image(xr::Duration::INFINITE)
+    }
+
+    fn release_image(&self) -> xr::Result<()> {
+        self.handle.lock().unwrap().release_image()
+    }
+
+    fn end(
+        &self,
+        predicted_display_time: xr::Time,
         views: &[openxr::View],
         stage: &xr::Space,
+        resolution: UVec2,
         environment_blend_mode: xr::EnvironmentBlendMode,
     ) -> xr::Result<()> {
-        self.handle.release_image().unwrap();
         let rect = xr::Rect2Di {
             offset: xr::Offset2Di { x: 0, y: 0 },
             extent: xr::Extent2Di {
-                    width: self.resolution.x as _,
+                width: resolution.x as _,
-                    height: self.resolution.y as _,
+                height: resolution.y as _,
             },
         };
-            self.stream.end(
+        let swapchain = self.handle.lock().unwrap();
-                xr_frame_state.predicted_display_time,
+        self.stream.lock().unwrap().end(
+            predicted_display_time,
             environment_blend_mode,
             &[&xr::CompositionLayerProjection::new().space(stage).views(&[
                 xr::CompositionLayerProjectionView::new()
@@ -126,7 +143,7 @@ impl<G: xr::Graphics> SwapchainInner<G> {
                     .fov(views[0].fov)
                     .sub_image(
                         xr::SwapchainSubImage::new()
-                                .swapchain(&self.handle)
+                            .swapchain(&swapchain)
                             .image_array_index(0)
                             .image_rect(rect),
                     ),
@@ -135,7 +152,7 @@ impl<G: xr::Graphics> SwapchainInner<G> {
                     .fov(views[1].fov)
                     .sub_image(
                         xr::SwapchainSubImage::new()
-                                .swapchain(&self.handle)
+                            .swapchain(&swapchain)
                             .image_array_index(1)
                             .image_rect(rect),
                     ),

src/xr_input/controllers.rs

@@ -0,0 +1,14 @@
+use openxr::{Action, ActionTy};
+
+pub struct Touchable<T: ActionTy> {
+    pub inner: Action<T>,
+    pub touch: Action<bool>,
+}
+pub struct Handed<T> {
+    pub left: T,
+    pub right: T,
+}
+#[derive(Copy, Clone)]
+pub enum XrControllerType {
+    OculusTouch,
+}

src/xr_input/mod.rs

@@ -0,0 +1,88 @@
+pub mod controllers;
+pub mod oculus_touch;
+pub mod xr_camera;
+
+use crate::resources::XrSession;
+use crate::xr_input::controllers::XrControllerType;
+use crate::xr_input::oculus_touch::{setup_oculus_controller, ActionSets};
+use crate::xr_input::xr_camera::{
+    xr_camera_head_sync, Eye, XRProjection, XrCameraBundle, XrCamerasBundle,
+};
+use bevy::app::{App, PostUpdate, Startup};
+use bevy::log::{info, warn};
+use bevy::prelude::IntoSystemConfigs;
+use bevy::prelude::{
+    Commands, Component, IntoSystemSetConfigs, Plugin, PreUpdate, Quat, Res, Resource, Vec3,
+};
+use bevy::render::camera::CameraProjectionPlugin;
+use bevy::render::view::{update_frusta, VisibilitySystems};
+use bevy::transform::TransformSystem;
+use openxr::{Action, ActionSet, ActionTy};
+
+#[derive(Copy, Clone)]
+pub struct OpenXrInput {
+    pub controller_type: XrControllerType,
+}
+impl OpenXrInput {
+    pub fn new(controller_type: XrControllerType) -> Self {
+        Self { controller_type }
+    }
+}
+
+impl Plugin for OpenXrInput {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(CameraProjectionPlugin::<XRProjection>::default());
+        match self.controller_type {
+            XrControllerType::OculusTouch => {
+                app.add_systems(Startup, setup_oculus_controller);
+            }
+        }
+        app.add_systems(PreUpdate, action_set_system);
+        app.add_systems(PreUpdate, xr_camera_head_sync);
+        app.add_systems(
+            PostUpdate,
+            update_frusta::<XRProjection>
+                .after(TransformSystem::TransformPropagate)
+                .before(VisibilitySystems::UpdatePerspectiveFrusta),
+        );
+        app.add_systems(Startup, setup_xr_cameras);
+    }
+}
+
+fn setup_xr_cameras(mut commands: Commands) {
+    commands.spawn(XrCameraBundle::new(Eye::Right));
+    commands.spawn(XrCameraBundle::new(Eye::Left));
+}
+
+fn action_set_system(action_sets: Res<ActionSets>, session: Res<XrSession>) {
+    let mut active_action_sets = vec![];
+    for i in &action_sets.0 {
+        active_action_sets.push(openxr::ActiveActionSet::new(i));
+    }
+    info!("action sets: {:#?}", action_sets.0.len());
+    match session.sync_actions(&active_action_sets) {
+        Err(err) => {
+            warn!("{}", err);
+        }
+        _ => {}
+    }
+}
+
+pub trait Vec3Conv {
+    fn to_vec3(&self) -> Vec3;
+}
+
+impl Vec3Conv for openxr::Vector3f {
+    fn to_vec3(&self) -> Vec3 {
+        Vec3::new(self.x, self.y, self.z)
+    }
+}
+pub trait QuatConv {
+    fn to_quat(&self) -> Quat;
+}
+
+impl QuatConv for openxr::Quaternionf {
+    fn to_quat(&self) -> Quat {
+        Quat::from_xyzw(self.x, self.y, self.z, self.w)
+    }
+}

src/xr_input/oculus_touch.rs

@@ -0,0 +1,254 @@
+use crate::resources::{XrInstance, XrSession};
+use crate::xr_input::controllers::{Handed, Touchable};
+use crate::FutureXrResources;
+use bevy::prelude::{Commands, Res, Resource};
+use openxr::{Action, ActionSet, AnyGraphics, Binding, Haptic, Instance, Posef, Session, Space};
+use std::any::Any;
+
+pub fn setup_oculus_controller(
+    mut commands: Commands,
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+) {
+    let mut action_sets = vec![];
+    let oculus_controller = OculusController::new(
+        Instance::clone(&instance),
+        Session::clone(&session),
+        &mut action_sets,
+    )
+    .unwrap();
+    session
+        .attach_action_sets(&action_sets.iter().map(|a| a).collect::<Vec<_>>())
+        .unwrap();
+    commands.insert_resource(oculus_controller);
+    commands.insert_resource(ActionSets(action_sets));
+}
+
+#[derive(Resource, Clone)]
+pub struct ActionSets(pub Vec<ActionSet>);
+
+#[derive(Resource)]
+pub struct OculusController {
+    pub grip_space: Handed<Space>,
+    pub aim_space: Handed<Space>,
+    pub grip_pose: Action<Posef>,
+    pub aim_pose: Action<Posef>,
+    pub squeeze: Action<f32>,
+    pub trigger: Touchable<f32>,
+    pub haptic_feedback: Action<Haptic>,
+    pub x_button: Touchable<bool>,
+    pub y_button: Touchable<bool>,
+    pub menu_button: Action<bool>,
+    pub a_button: Touchable<bool>,
+    pub b_button: Touchable<bool>,
+    pub thumbstick_x: Action<f32>,
+    pub thumbstick_y: Action<f32>,
+    pub thumbstick_touch: Action<bool>,
+    pub thumbstick_click: Action<bool>,
+    pub thumbrest_touch: Action<bool>,
+}
+impl OculusController {
+    pub fn new(
+        instance: Instance,
+        session: Session<AnyGraphics>,
+        action_sets: &mut Vec<ActionSet>,
+    ) -> anyhow::Result<Self> {
+        let action_set =
+            instance.create_action_set("oculus_input", "Oculus Touch Controller Input", 0)?;
+        let left_path = instance.string_to_path("/user/hand/left").unwrap();
+        let right_path = instance.string_to_path("/user/hand/right").unwrap();
+        let hands = [left_path, right_path];
+        let grip_pose = action_set.create_action::<Posef>("hand_pose", "Hand Pose", &hands)?;
+        let aim_pose = action_set.create_action::<Posef>("pointer_pose", "Pointer Pose", &hands)?;
+
+        let this = OculusController {
+            grip_space: Handed {
+                left: grip_pose.create_space(session.clone(), right_path, Posef::IDENTITY)?,
+                right: grip_pose.create_space(session.clone(), left_path, Posef::IDENTITY)?,
+            },
+            aim_space: Handed {
+                left: aim_pose.create_space(session.clone(), right_path, Posef::IDENTITY)?,
+                right: aim_pose.create_space(session.clone(), left_path, Posef::IDENTITY)?,
+            },
+            grip_pose,
+            aim_pose,
+            squeeze: action_set.create_action("squeeze", "Grip Pull", &hands)?,
+            trigger: Touchable {
+                inner: action_set.create_action("trigger", "Trigger Pull", &hands)?,
+                touch: action_set.create_action("trigger_touched", "Trigger Touch", &hands)?,
+            },
+            haptic_feedback: action_set.create_action(
+                "haptic_feedback",
+                "Haptic Feedback",
+                &hands,
+            )?,
+            x_button: Touchable {
+                inner: action_set.create_action("x_button", "X Button", &[])?,
+                touch: action_set.create_action("x_button_touch", "X Button Touch", &[])?,
+            },
+            y_button: Touchable {
+                inner: action_set.create_action("y_button", "Y Button", &[])?,
+                touch: action_set.create_action("y_button_touch", "Y Button Touch", &[])?,
+            },
+            menu_button: action_set.create_action("menu_button", "Menu Button", &[])?,
+            a_button: Touchable {
+                inner: action_set.create_action("a_button", "A Button", &[])?,
+                touch: action_set.create_action("a_button_touch", "A Button Touch", &[])?,
+            },
+            b_button: Touchable {
+                inner: action_set.create_action("b_button", "B Button", &[])?,
+                touch: action_set.create_action("b_button_touch", "B Button Touch", &[])?,
+            },
+            thumbstick_x: action_set.create_action("thumbstick_x", "Thumbstick X", &hands)?,
+            thumbstick_y: action_set.create_action("thumbstick_y", "Thumbstick Y", &hands)?,
+            thumbstick_touch: action_set.create_action(
+                "thumbstick_touch",
+                "Thumbstick Touch",
+                &hands,
+            )?,
+            thumbstick_click: action_set.create_action(
+                "thumbstick_click",
+                "Thumbstick Click",
+                &hands,
+            )?,
+            thumbrest_touch: action_set.create_action(
+                "thumbrest_touch",
+                "Thumbrest Touch",
+                &hands,
+            )?,
+        };
+        let i = instance;
+        i.suggest_interaction_profile_bindings(
+            i.string_to_path("/interaction_profiles/oculus/touch_controller")?,
+            &[
+                Binding::new(
+                    &this.grip_pose,
+                    i.string_to_path("/user/hand/left/input/grip/pose")?,
+                ),
+                Binding::new(
+                    &this.grip_pose,
+                    i.string_to_path("/user/hand/right/input/grip/pose")?,
+                ),
+                Binding::new(
+                    &this.aim_pose,
+                    i.string_to_path("/user/hand/left/input/aim/pose")?,
+                ),
+                Binding::new(
+                    &this.aim_pose,
+                    i.string_to_path("/user/hand/left/input/aim/pose")?,
+                ),
+                Binding::new(
+                    &this.squeeze,
+                    i.string_to_path("/user/hand/left/input/squeeze/value")?,
+                ),
+                Binding::new(
+                    &this.squeeze,
+                    i.string_to_path("/user/hand/right/input/squeeze/value")?,
+                ),
+                Binding::new(
+                    &this.trigger.inner,
+                    i.string_to_path("/user/hand/right/input/trigger/value")?,
+                ),
+                Binding::new(
+                    &this.trigger.inner,
+                    i.string_to_path("/user/hand/left/input/trigger/value")?,
+                ),
+                Binding::new(
+                    &this.trigger.touch,
+                    i.string_to_path("/user/hand/right/input/trigger/touch")?,
+                ),
+                Binding::new(
+                    &this.trigger.touch,
+                    i.string_to_path("/user/hand/left/input/trigger/touch")?,
+                ),
+                Binding::new(
+                    &this.haptic_feedback,
+                    i.string_to_path("/user/hand/right/output/haptic")?,
+                ),
+                Binding::new(
+                    &this.haptic_feedback,
+                    i.string_to_path("/user/hand/left/output/haptic")?,
+                ),
+                Binding::new(
+                    &this.x_button.inner,
+                    i.string_to_path("/user/hand/left/input/x/click")?,
+                ),
+                Binding::new(
+                    &this.x_button.touch,
+                    i.string_to_path("/user/hand/left/input/x/touch")?,
+                ),
+                Binding::new(
+                    &this.y_button.inner,
+                    i.string_to_path("/user/hand/left/input/y/click")?,
+                ),
+                Binding::new(
+                    &this.y_button.touch,
+                    i.string_to_path("/user/hand/left/input/y/touch")?,
+                ),
+                Binding::new(
+                    &this.menu_button,
+                    i.string_to_path("/user/hand/left/input/menu/click")?,
+                ),
+                Binding::new(
+                    &this.a_button.inner,
+                    i.string_to_path("/user/hand/right/input/a/click")?,
+                ),
+                Binding::new(
+                    &this.a_button.touch,
+                    i.string_to_path("/user/hand/right/input/a/touch")?,
+                ),
+                Binding::new(
+                    &this.b_button.inner,
+                    i.string_to_path("/user/hand/right/input/b/click")?,
+                ),
+                Binding::new(
+                    &this.b_button.touch,
+                    i.string_to_path("/user/hand/right/input/b/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_x,
+                    i.string_to_path("/user/hand/left/input/thumbstick/x")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_x,
+                    i.string_to_path("/user/hand/right/input/thumbstick/x")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_y,
+                    i.string_to_path("/user/hand/left/input/thumbstick/y")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_y,
+                    i.string_to_path("/user/hand/right/input/thumbstick/y")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_click,
+                    i.string_to_path("/user/hand/left/input/thumbstick/click")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_click,
+                    i.string_to_path("/user/hand/right/input/thumbstick/click")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_touch,
+                    i.string_to_path("/user/hand/left/input/thumbstick/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_touch,
+                    i.string_to_path("/user/hand/right/input/thumbstick/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbrest_touch,
+                    i.string_to_path("/user/hand/left/input/thumbrest/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbrest_touch,
+                    i.string_to_path("/user/hand/right/input/thumbrest/touch")?,
+                ),
+            ],
+        )?;
+
+        action_sets.push(action_set);
+        Ok(this)
+    }
+}

src/xr_input/xr_camera.rs

@@ -0,0 +1,239 @@
+use crate::xr_input::{QuatConv, Vec3Conv};
+use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
+use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping};
+use bevy::prelude::*;
+use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget};
+use bevy::render::primitives::Frustum;
+use bevy::render::view::{ColorGrading, VisibleEntities};
+use openxr::Fovf;
+
+#[derive(Bundle)]
+pub struct XrCamerasBundle {
+    pub left: XrCameraBundle,
+    pub right: XrCameraBundle,
+}
+impl XrCamerasBundle {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+impl Default for XrCamerasBundle {
+    fn default() -> Self {
+        Self {
+            left: XrCameraBundle::new(Eye::Left),
+            right: XrCameraBundle::new(Eye::Right),
+        }
+    }
+}
+
+#[derive(Bundle)]
+pub struct XrCameraBundle {
+    pub camera: Camera,
+    pub camera_render_graph: CameraRenderGraph,
+    pub xr_projection: XRProjection,
+    pub visible_entities: VisibleEntities,
+    pub frustum: Frustum,
+    pub transform: Transform,
+    pub global_transform: GlobalTransform,
+    pub camera_3d: Camera3d,
+    pub tonemapping: Tonemapping,
+    pub dither: DebandDither,
+    pub color_grading: ColorGrading,
+    pub xr_camera_type: XrCameraType,
+}
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)]
+pub enum XrCameraType {
+    Xr(Eye),
+    Flatscreen,
+}
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
+pub enum Eye {
+    Left = 0,
+    Right = 1,
+}
+
+impl XrCameraBundle {
+    pub fn new(eye: Eye) -> Self {
+        Self {
+            camera: Camera {
+                order: -1,
+                target: RenderTarget::TextureView(match eye {
+                    Eye::Left => LEFT_XR_TEXTURE_HANDLE,
+                    Eye::Right => RIGHT_XR_TEXTURE_HANDLE,
+                }),
+                viewport: None,
+                ..default()
+            },
+            camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME),
+            xr_projection: Default::default(),
+            visible_entities: Default::default(),
+            frustum: Default::default(),
+            transform: Default::default(),
+            global_transform: Default::default(),
+            camera_3d: Default::default(),
+            tonemapping: Default::default(),
+            dither: DebandDither::Enabled,
+            color_grading: Default::default(),
+            xr_camera_type: XrCameraType::Xr(eye),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Component, Reflect)]
+#[reflect(Component, Default)]
+pub struct XRProjection {
+    pub near: f32,
+    pub far: f32,
+    #[reflect(ignore)]
+    pub fov: Fovf,
+}
+
+impl Default for XRProjection {
+    fn default() -> Self {
+        Self {
+            near: 0.1,
+            far: 1000.,
+            fov: Default::default(),
+        }
+    }
+}
+
+impl XRProjection {
+    pub fn new(near: f32, far: f32, fov: Fovf) -> Self {
+        XRProjection { near, far, fov }
+    }
+}
+
+impl CameraProjection for XRProjection {
+    // =============================================================================
+    // math code adapted from
+    // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h
+    // Copyright (c) 2017 The Khronos Group Inc.
+    // Copyright (c) 2016 Oculus VR, LLC.
+    // SPDX-License-Identifier: Apache-2.0
+    // =============================================================================
+    fn get_projection_matrix(&self) -> Mat4 {
+        //  symmetric perspective for debugging
+        // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs());
+        // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs());
+        // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near);
+
+        let fov = self.fov;
+        let is_vulkan_api = false; // FIXME wgpu probably abstracts this
+        let near_z = self.near;
+        let far_z = -1.; //   use infinite proj
+                         // let far_z = self.far;
+
+        let tan_angle_left = fov.angle_left.tan();
+        let tan_angle_right = fov.angle_right.tan();
+
+        let tan_angle_down = fov.angle_down.tan();
+        let tan_angle_up = fov.angle_up.tan();
+
+        let tan_angle_width = tan_angle_right - tan_angle_left;
+
+        // Set to tanAngleDown - tanAngleUp for a clip space with positive Y
+        // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with
+        // positive Y up (OpenGL / D3D / Metal).
+        // const float tanAngleHeight =
+        //     graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown);
+        let tan_angle_height = if is_vulkan_api {
+            tan_angle_down - tan_angle_up
+        } else {
+            tan_angle_up - tan_angle_down
+        };
+
+        // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES).
+        // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal).
+        // const float offsetZ =
+        //     (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0;
+        // FIXME handle enum of graphics apis
+        let offset_z = 0.;
+
+        let mut cols: [f32; 16] = [0.0; 16];
+
+        if far_z <= near_z {
+            // place the far plane at infinity
+            cols[0] = 2. / tan_angle_width;
+            cols[4] = 0.;
+            cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width;
+            cols[12] = 0.;
+
+            cols[1] = 0.;
+            cols[5] = 2. / tan_angle_height;
+            cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height;
+            cols[13] = 0.;
+
+            cols[2] = 0.;
+            cols[6] = 0.;
+            cols[10] = -1.;
+            cols[14] = -(near_z + offset_z);
+
+            cols[3] = 0.;
+            cols[7] = 0.;
+            cols[11] = -1.;
+            cols[15] = 0.;
+
+            //  bevy uses the _reverse_ infinite projection
+            //  https://dev.theomader.com/depth-precision/
+            let z_reversal = Mat4::from_cols_array_2d(&[
+                [1f32, 0., 0., 0.],
+                [0., 1., 0., 0.],
+                [0., 0., -1., 0.],
+                [0., 0., 1., 1.],
+            ]);
+
+            return z_reversal * Mat4::from_cols_array(&cols);
+        } else {
+            // normal projection
+            cols[0] = 2. / tan_angle_width;
+            cols[4] = 0.;
+            cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width;
+            cols[12] = 0.;
+
+            cols[1] = 0.;
+            cols[5] = 2. / tan_angle_height;
+            cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height;
+            cols[13] = 0.;
+
+            cols[2] = 0.;
+            cols[6] = 0.;
+            cols[10] = -(far_z + offset_z) / (far_z - near_z);
+            cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z);
+
+            cols[3] = 0.;
+            cols[7] = 0.;
+            cols[11] = -1.;
+            cols[15] = 0.;
+        }
+
+        Mat4::from_cols_array(&cols)
+    }
+
+    fn update(&mut self, _width: f32, _height: f32) {}
+
+    fn far(&self) -> f32 {
+        self.far
+    }
+}
+
+pub fn xr_camera_head_sync(
+    views: ResMut<crate::resources::XrViews>,
+    mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>,
+) {
+    let mut f = || -> Option<()> {
+        for (mut transform, camera_type, mut xr_projection) in query.iter_mut() {
+            let view_idx = match camera_type {
+                XrCameraType::Xr(eye) => *eye as usize,
+                XrCameraType::Flatscreen => return None,
+            };
+            let v = views.lock().unwrap();
+            let view = v.get(view_idx)?;
+            xr_projection.fov = view.fov;
+            transform.rotation = view.pose.orientation.to_quat();
+            transform.translation = view.pose.position.to_vec3();
+        }
+        Some(())
+    };
+    let _ = f();
+}