input refactor

Cargo.toml

@@ -9,7 +9,8 @@ edition = "2021"
 anyhow = "1.0.75"
 ash = "0.37.3"
 bevy = { git = "https://github.com/awtterpip/bevy", default-features = false, features = ["bevy_render"] }
-openxr = { version = "0.17.1" }
+openxr = { version = "0.17.1", features = ["mint"] }
+mint = "0.5.9"
 wgpu = "0.16.0"
 wgpu-core = { version = "0.16.0", features = ["vulkan"] }
 wgpu-hal = "0.16.0"
@@ -21,3 +22,6 @@ color-eyre = "0.6.2"
 [[example]]
 name = "xr"
 path = "examples/xr.rs"
+
+[profile.release]
+debug = true

examples/xr.rs

@@ -193,6 +193,9 @@ use bevy::transform::TransformSystem;
 use bevy::{prelude::*, render::camera::RenderTarget};
 use bevy_openxr::input::XrInput;
 use bevy_openxr::resources::{XrFrameState, XrSession, XrViews};
+use bevy_openxr::xr_input::controllers::XrControllerType;
+use bevy_openxr::xr_input::oculus_touch::OculusController;
+use bevy_openxr::xr_input::OpenXrInput;
 use bevy_openxr::{DefaultXrPlugins, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
 use openxr::ActiveActionSet;
 
@@ -202,16 +205,9 @@ fn main() {
     info!("Running `openxr-6dof` skill");
     App::new()
         .add_plugins(DefaultXrPlugins)
-		.add_plugins(CameraProjectionPlugin::<XRProjection>::default())
+        .add_plugins(OpenXrInput::new(XrControllerType::OculusTouch))
         .add_systems(Startup, setup)
-		.add_systems(PreUpdate, head_movement)
+        .add_systems(Update, hands)
-		.add_systems(PreUpdate, hands)
-		.add_systems(
-			PostUpdate,
-			update_frusta::<XRProjection>
-				.after(TransformSystem::TransformPropagate)
-				.before(VisibilitySystems::UpdatePerspectiveFrusta),
-		)
         .run();
 }
 
@@ -254,44 +250,46 @@ fn setup(
     // camera
     commands.spawn((
         Camera3dBundle {
-			transform: Transform::from_xyz(-2.0, 2.5, 5.0)
+            transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y),
-				.looking_at(Vec3::ZERO, Vec3::Y),
             ..default()
         },
         CameraType::Middle,
     ));
-
-	commands.spawn((
-		XrCameraBundle {
-			transform: Transform::from_xyz(-2.0, 2.5, 5.0)
-				.looking_at(Vec3::ZERO, Vec3::Y),
-			camera: Camera {
-				order: -1,
-				target: RenderTarget::TextureView(LEFT_XR_TEXTURE_HANDLE),
-				viewport: None,
-				..default()
-			},
-			..default()
-		},
-		CameraType::Left,
-	));
-	commands.spawn((
-		XrCameraBundle {
-			transform: Transform::from_xyz(-2.0, 2.5, 5.0)
-				.looking_at(Vec3::ZERO, Vec3::Y),
-			camera: Camera {
-				order: -1,
-				target: RenderTarget::TextureView(RIGHT_XR_TEXTURE_HANDLE),
-				viewport: None,
-				..default()
-			},
-			..default()
-		},
-		CameraType::Right,
-	));
 }
 
 fn hands(
+    mut gizmos: Gizmos,
+    oculus_controller: Res<OculusController>,
+    frame_state: Res<XrFrameState>,
+    xr_input: Res<XrInput>,
+) {
+    frame_state.lock().unwrap().map(|a| {
+        let right_controller = oculus_controller
+            .grip_space
+            .right
+            .relate(&**&xr_input.stage, a.predicted_display_time)
+            .unwrap();
+        let left_controller = oculus_controller
+            .grip_space
+            .left
+            .relate(&**&xr_input.stage, a.predicted_display_time)
+            .unwrap();
+        gizmos.rect(
+            right_controller.0.pose.position.to_vec3(),
+            right_controller.0.pose.orientation.to_quat(),
+            Vec2::new(0.05, 0.2),
+            Color::YELLOW_GREEN,
+        );
+        gizmos.rect(
+            left_controller.0.pose.position.to_vec3(),
+            left_controller.0.pose.orientation.to_quat(),
+            Vec2::new(0.05, 0.2),
+            Color::YELLOW_GREEN,
+        );
+    });
+}
+
+/*fn hands(
     mut gizmos: Gizmos,
     xr_input: Res<XrInput>,
     session: Res<XrSession>,
@@ -323,7 +321,7 @@ fn hands(
             Color::YELLOW_GREEN,
         )
     });
-}
+}*/
 
 fn head_movement(
     views: ResMut<XrViews>,
@@ -331,17 +329,12 @@ fn head_movement(
 ) {
     let views = views.lock().unwrap();
     let mut f = || -> Option<()> {
-		let midpoint = (views.get(0)?.pose.position.to_vec3()
+        let midpoint =
-			+ views.get(1)?.pose.position.to_vec3())
+            (views.get(0)?.pose.position.to_vec3() + views.get(1)?.pose.position.to_vec3()) / 2.;
-			/ 2.;
         for (mut t, _, camera_type, _) in query.iter_mut() {
             match camera_type {
-				CameraType::Left => {
+                CameraType::Left => t.translation = views.get(0)?.pose.position.to_vec3(),
-					t.translation = views.get(0)?.pose.position.to_vec3()
+                CameraType::Right => t.translation = views.get(1)?.pose.position.to_vec3(),
-				}
-				CameraType::Right => {
-					t.translation = views.get(1)?.pose.position.to_vec3()
-				}
                 CameraType::Middle => {
                     t.translation = midpoint;
                 }

src/graphics/mod.rs

@@ -1,12 +1,32 @@
 mod vulkan;
 
-use bevy::render::renderer::{RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter};
+use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue};
 use bevy::window::RawHandleWrapper;
 use wgpu::Instance;
 
 use crate::input::XrInput;
-use crate::resources::{XrInstance, XrSession, XrEnvironmentBlendMode, XrSessionRunning, XrFrameWaiter, XrSwapchain, XrViews, XrFrameState};
+use crate::resources::{
+    XrEnvironmentBlendMode, XrFrameState, XrFrameWaiter, XrInstance, XrSession, XrSessionRunning,
+    XrSwapchain, XrViews,
+};
 
-pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Result<(RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter, Instance, XrInstance, XrSession, XrEnvironmentBlendMode, XrSessionRunning, XrFrameWaiter, XrSwapchain, XrInput, XrViews, XrFrameState)>{
+pub fn initialize_xr_graphics(
+    window: Option<RawHandleWrapper>,
+) -> anyhow::Result<(
+    RenderDevice,
+    RenderQueue,
+    RenderAdapterInfo,
+    RenderAdapter,
+    Instance,
+    XrInstance,
+    XrSession,
+    XrEnvironmentBlendMode,
+    XrSessionRunning,
+    XrFrameWaiter,
+    XrSwapchain,
+    XrInput,
+    XrViews,
+    XrFrameState,
+)> {
     vulkan::initialize_xr_graphics(window)
 }

src/graphics/vulkan.rs

@@ -1,6 +1,6 @@
 use std::ffi::{c_void, CString};
-use std::sync::{Arc, Mutex};
 use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, Mutex};
 
 use anyhow::Context;
 use ash::vk::{self, Handle};
@@ -13,11 +13,14 @@ use wgpu::{Instance, Texture};
 
 use crate::input::XrInput;
 use crate::resources::{
-    XrEnvironmentBlendMode, XrFrameWaiter, XrInstance, XrSession, XrSessionRunning, XrSwapchain, Swapchain, SwapchainInner, XrViews, XrFrameState,
+    Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFrameState, XrFrameWaiter, XrInstance,
+    XrSession, XrSessionRunning, XrSwapchain, XrViews,
 };
 use crate::VIEW_TYPE;
 
-pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Result<(
+pub fn initialize_xr_graphics(
+    window: Option<RawHandleWrapper>,
+) -> anyhow::Result<(
     RenderDevice,
     RenderQueue,
     RenderAdapterInfo,
@@ -281,7 +284,10 @@ pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Resul
         .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0])
         .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb);
 
-    let resolution = uvec2(views[0].recommended_image_rect_width, views[0].recommended_image_rect_height);
+    let resolution = uvec2(
+        views[0].recommended_image_rect_width,
+        views[0].recommended_image_rect_height,
+    );
 
     let handle = session
         .create_swapchain(&xr::SwapchainCreateInfo {
@@ -370,7 +376,8 @@ pub fn initialize_xr_graphics(window: Option<RawHandleWrapper>) -> anyhow::Resul
             format: swapchain_format,
             buffers,
             image_index: 0,
-        })).into(),
+        }))
+        .into(),
         XrInput::new(xr_instance, session.into_any_graphics())?,
         Mutex::default().into(),
         Mutex::default().into(),

src/input.rs

@@ -7,57 +7,58 @@ type XrPose = (Vec3, Quat);
 
 #[derive(Clone, Resource)]
 pub struct XrInput {
-    pub action_set: xr::ActionSet,
+    //pub action_set: xr::ActionSet,
-    pub hand_pose: xr::Action<xr::Posef>,
+    //pub hand_pose: xr::Action<xr::Posef>,
-    pub right_space: Arc<xr::Space>,
+    //pub right_space: Arc<xr::Space>,
-    pub left_space: Arc<xr::Space>,
+    //pub left_space: Arc<xr::Space>,
     pub stage: Arc<xr::Space>,
 }
 
 impl XrInput {
-    pub fn new(
+    pub fn new(instance: xr::Instance, session: xr::Session<xr::AnyGraphics>) -> xr::Result<Self> {
-        instance: xr::Instance,
+        // let action_set = instance.create_action_set("input", "input pose information", 0)?;
-        session: xr::Session<xr::AnyGraphics>,
+        // let left_hand_subaction_path = instance.string_to_path("/user/hand/left").unwrap();
-    ) -> xr::Result<Self> {
+        // let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap();
-        let action_set = instance.create_action_set("input", "input pose information", 0)?;
+        // let left_hand_grip_pose_path = instance
-        let left_hand_subaction_path = instance.string_to_path("/user/hand/left").unwrap();
+        //     .string_to_path("/user/hand/left/input/grip/pose")
-        let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap();
+        //     .unwrap();
-        let left_hand_grip_pose_path = instance
+        // let right_hand_grip_pose_path = instance
-            .string_to_path("/user/hand/left/input/grip/pose")
+        //     .string_to_path("/user/hand/right/input/grip/pose")
-            .unwrap();
+        //     .unwrap();
-        let right_hand_grip_pose_path = instance
+        // let hand_pose = action_set.create_action::<xr::Posef>(
-            .string_to_path("/user/hand/right/input/grip/pose")
+        //     "hand_pose",
-            .unwrap();
+        //     "Hand Pose",
-        let hand_pose =
+        //     &[left_hand_subaction_path, right_hand_subaction_path],
-            action_set.create_action::<xr::Posef>("hand_pose", "Hand Pose", &[left_hand_subaction_path, right_hand_subaction_path])?;
+        // )?;
-       /* let left_action =
+        // /* let left_action =
-            action_set.create_action::<xr::Posef>("left_hand", "Left Hand Controller", &[])?;*/
+        // action_set.create_action::<xr::Posef>("left_hand", "Left Hand Controller", &[])?;*/
-        instance.suggest_interaction_profile_bindings(
+        // instance.suggest_interaction_profile_bindings(
-            instance.string_to_path("/interaction_profiles/khr/simple_controller")?,
+        //     instance.string_to_path("/interaction_profiles/khr/simple_controller")?,
-            &[
+        //     &[
-                xr::Binding::new(
+        //         xr::Binding::new(&hand_pose, right_hand_grip_pose_path),
-                    &hand_pose,
+        //         xr::Binding::new(&hand_pose, left_hand_grip_pose_path),
-                    right_hand_grip_pose_path,
+        //     ],
-                ),
+        // )?;
-                xr::Binding::new(
+        //
-                    &hand_pose,
+        // let right_space = hand_pose.create_space(
-                    left_hand_grip_pose_path,
+        //     session.clone(),
-                ),
+        //     right_hand_subaction_path,
-            ],
+        //     xr::Posef::IDENTITY,
-        )?;
+        // )?;
-
+        // let left_space = hand_pose.create_space(
-        let right_space =
+        //     session.clone(),
-            hand_pose.create_space(session.clone(), right_hand_subaction_path, xr::Posef::IDENTITY)?;
+        //     left_hand_subaction_path,
-        let left_space =
+        //     xr::Posef::IDENTITY,
-            hand_pose.create_space(session.clone(), left_hand_subaction_path, xr::Posef::IDENTITY)?;
+        // )?;
         let stage =
             session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?;
-        session.attach_action_sets(&[&action_set])?;
+        //session.attach_action_sets(&[&action_set])?;
+        //session.attach_action_sets(&[])?;
         Ok(Self {
-            action_set,
+            //action_set,
-            hand_pose,
+            //hand_pose,
-            right_space: Arc::new(right_space),
+            // right_space: Arc::new(right_space),
-            left_space: Arc::new(left_space),
+            // left_space: Arc::new(left_space),
             stage: Arc::new(stage),
         })
     }

src/lib.rs

@@ -2,14 +2,16 @@ mod graphics;
 pub mod input;
 pub mod resource_macros;
 pub mod resources;
+pub mod xr_input;
 
 use std::sync::{Arc, Mutex};
 
+use crate::xr_input::oculus_touch::ActionSets;
 use bevy::app::PluginGroupBuilder;
 use bevy::ecs::system::SystemState;
 use bevy::prelude::*;
 use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews};
-use bevy::render::renderer::{RenderAdapterInfo, RenderAdapter, RenderDevice, RenderQueue};
+use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue};
 use bevy::render::settings::RenderSettings;
 use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet};
 use bevy::window::{PrimaryWindow, RawHandleWrapper};
@@ -85,7 +87,16 @@ impl Plugin for OpenXrPlugin {
             views,
             frame_state,
         ));
-        app.add_plugins(DefaultPlugins.set(RenderPlugin { render_settings: RenderSettings::Manual(device, queue, adapter_info, render_adapter, Mutex::new(instance))}));
+        app.insert_resource(ActionSets(vec![]));
+        app.add_plugins(DefaultPlugins.set(RenderPlugin {
+            render_settings: RenderSettings::Manual(
+                device,
+                queue,
+                adapter_info,
+                render_adapter,
+                Mutex::new(instance),
+            ),
+        }));
     }
 
     fn ready(&self, app: &App) -> bool {
@@ -109,6 +120,8 @@ impl Plugin for OpenXrPlugin {
                 frame_state,
             ) = future_renderer_resources.0.lock().unwrap().take().unwrap();
 
+            let action_sets = app.world.resource::<ActionSets>().clone();
+
             app.insert_resource(xr_instance.clone())
                 .insert_resource(session.clone())
                 .insert_resource(blend_mode.clone())
@@ -117,7 +130,8 @@ impl Plugin for OpenXrPlugin {
                 .insert_resource(swapchain.clone())
                 .insert_resource(input.clone())
                 .insert_resource(views.clone())
-                .insert_resource(frame_state.clone());
+                .insert_resource(frame_state.clone())
+                .insert_resource(action_sets.clone());
 
             let swapchain_mut = swapchain.lock().unwrap();
             let (left, right) = swapchain_mut.get_render_views();
@@ -148,7 +162,8 @@ impl Plugin for OpenXrPlugin {
                 .insert_resource(swapchain)
                 .insert_resource(input)
                 .insert_resource(views)
-                .insert_resource(frame_state);
+                .insert_resource(frame_state)
+                .insert_resource(action_sets);
 
             render_app.add_systems(
                 Render,
@@ -180,6 +195,7 @@ pub fn pre_frame(
     frame_waiter: Res<XrFrameWaiter>,
     swapchain: Res<XrSwapchain>,
     xr_input: Res<XrInput>,
+    action_sets: Res<ActionSets>,
     mut manual_texture_views: ResMut<ManualTextureViews>,
 ) {
     while let Some(event) = instance.poll_event(&mut Default::default()).unwrap() {
@@ -211,7 +227,7 @@ pub fn pre_frame(
     }
     if !session_running.load(std::sync::atomic::Ordering::Relaxed) {
         // Don't grind up the CPU
-        std::thread::sleep(std::time::Duration::from_millis(10));
+        //std::thread::sleep(std::time::Duration::from_millis(10));
         return;
     }
 
@@ -222,13 +238,17 @@ pub fn pre_frame(
     swapchain.begin().unwrap();
     swapchain.update_render_views();
     let (left, right) = swapchain.get_render_views();
-    let active_action_set = xr::ActiveActionSet::new(&xr_input.action_set);
+    /*let mut active_action_sets = vec![];
-    match session.sync_actions(&[active_action_set]) {
+    for i in &action_sets.0 {
+        active_action_sets.push(xr::ActiveActionSet::new(i));
+    }
+    info!("action sets: {:#?}", action_sets.0.len());
+    match session.sync_actions(&active_action_sets) {
         Err(err) => {
-            eprintln!("{}", err);
+            warn!("{}", err);
         }
         _ => {}
-    }
+    }*/
     let format = swapchain.format();
     let left = ManualTextureView {
         texture_view: left.into(),

src/resource_macros.rs

@@ -23,7 +23,7 @@ macro_rules! xr_resource_wrapper {
                 Self::new(value)
             }
         }
-    }
+    };
 }
 
 #[macro_export]
@@ -51,8 +51,8 @@ macro_rules! xr_arc_resource_wrapper {
                 Self::new(value)
             }
         }
-    }
+    };
 }
 
-pub use xr_resource_wrapper;
 pub use xr_arc_resource_wrapper;
+pub use xr_resource_wrapper;

src/resources.rs

@@ -40,7 +40,7 @@ impl Swapchain {
 
     pub(crate) fn format(&self) -> wgpu::TextureFormat {
         match self {
-            Swapchain::Vulkan(swap) => swap.format
+            Swapchain::Vulkan(swap) => swap.format,
         }
     }
 
@@ -58,7 +58,9 @@ impl Swapchain {
         environment_blend_mode: xr::EnvironmentBlendMode,
     ) -> xr::Result<()> {
         match self {
-            Swapchain::Vulkan(swap) => swap.post_queue_submit(xr_frame_state, views, stage, environment_blend_mode),
+            Swapchain::Vulkan(swap) => {
+                swap.post_queue_submit(xr_frame_state, views, stage, environment_blend_mode)
+            }
         }
     }
 }

src/xr_input/controllers.rs

@@ -0,0 +1,14 @@
+use openxr::{Action, ActionTy};
+
+pub struct Touchable<T: ActionTy> {
+    pub inner: Action<T>,
+    pub touch: Action<bool>,
+}
+pub struct Handed<T> {
+    pub left: T,
+    pub right: T,
+}
+#[derive(Copy, Clone)]
+pub enum XrControllerType {
+    OculusTouch,
+}

src/xr_input/mod.rs

@@ -0,0 +1,88 @@
+pub mod controllers;
+pub mod oculus_touch;
+pub mod xr_camera;
+
+use crate::resources::XrSession;
+use crate::xr_input::controllers::XrControllerType;
+use crate::xr_input::oculus_touch::{setup_oculus_controller, ActionSets};
+use crate::xr_input::xr_camera::{
+    xr_camera_head_sync, Eye, XRProjection, XrCameraBundle, XrCamerasBundle,
+};
+use bevy::app::{App, PostUpdate, Startup};
+use bevy::log::{info, warn};
+use bevy::prelude::IntoSystemConfigs;
+use bevy::prelude::{
+    Commands, Component, IntoSystemSetConfigs, Plugin, PreUpdate, Quat, Res, Resource, Vec3,
+};
+use bevy::render::camera::CameraProjectionPlugin;
+use bevy::render::view::{update_frusta, VisibilitySystems};
+use bevy::transform::TransformSystem;
+use openxr::{Action, ActionSet, ActionTy};
+
+#[derive(Copy, Clone)]
+pub struct OpenXrInput {
+    pub controller_type: XrControllerType,
+}
+impl OpenXrInput {
+    pub fn new(controller_type: XrControllerType) -> Self {
+        Self { controller_type }
+    }
+}
+
+impl Plugin for OpenXrInput {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(CameraProjectionPlugin::<XRProjection>::default());
+        match self.controller_type {
+            XrControllerType::OculusTouch => {
+                app.add_systems(Startup, setup_oculus_controller);
+            }
+        }
+        app.add_systems(PreUpdate, action_set_system);
+        app.add_systems(PreUpdate, xr_camera_head_sync);
+        app.add_systems(
+            PostUpdate,
+            update_frusta::<XRProjection>
+                .after(TransformSystem::TransformPropagate)
+                .before(VisibilitySystems::UpdatePerspectiveFrusta),
+        );
+        app.add_systems(Startup, setup_xr_cameras);
+    }
+}
+
+fn setup_xr_cameras(mut commands: Commands) {
+    commands.spawn(XrCameraBundle::new(Eye::Right));
+    commands.spawn(XrCameraBundle::new(Eye::Left));
+}
+
+fn action_set_system(action_sets: Res<ActionSets>, session: Res<XrSession>) {
+    let mut active_action_sets = vec![];
+    for i in &action_sets.0 {
+        active_action_sets.push(openxr::ActiveActionSet::new(i));
+    }
+    info!("action sets: {:#?}", action_sets.0.len());
+    match session.sync_actions(&active_action_sets) {
+        Err(err) => {
+            warn!("{}", err);
+        }
+        _ => {}
+    }
+}
+
+pub trait Vec3Conv {
+    fn to_vec3(&self) -> Vec3;
+}
+
+impl Vec3Conv for openxr::Vector3f {
+    fn to_vec3(&self) -> Vec3 {
+        Vec3::new(self.x, self.y, self.z)
+    }
+}
+pub trait QuatConv {
+    fn to_quat(&self) -> Quat;
+}
+
+impl QuatConv for openxr::Quaternionf {
+    fn to_quat(&self) -> Quat {
+        Quat::from_xyzw(self.x, self.y, self.z, self.w)
+    }
+}

src/xr_input/oculus_touch.rs

@@ -0,0 +1,254 @@
+use crate::resources::{XrInstance, XrSession};
+use crate::xr_input::controllers::{Handed, Touchable};
+use crate::FutureXrResources;
+use bevy::prelude::{Commands, Res, Resource};
+use openxr::{Action, ActionSet, AnyGraphics, Binding, Haptic, Instance, Posef, Session, Space};
+use std::any::Any;
+
+pub fn setup_oculus_controller(
+    mut commands: Commands,
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+) {
+    let mut action_sets = vec![];
+    let oculus_controller = OculusController::new(
+        Instance::clone(&instance),
+        Session::clone(&session),
+        &mut action_sets,
+    )
+    .unwrap();
+    session
+        .attach_action_sets(&action_sets.iter().map(|a| a).collect::<Vec<_>>())
+        .unwrap();
+    commands.insert_resource(oculus_controller);
+    commands.insert_resource(ActionSets(action_sets));
+}
+
+#[derive(Resource, Clone)]
+pub struct ActionSets(pub Vec<ActionSet>);
+
+#[derive(Resource)]
+pub struct OculusController {
+    pub grip_space: Handed<Space>,
+    pub aim_space: Handed<Space>,
+    pub grip_pose: Action<Posef>,
+    pub aim_pose: Action<Posef>,
+    pub squeeze: Action<f32>,
+    pub trigger: Touchable<f32>,
+    pub haptic_feedback: Action<Haptic>,
+    pub x_button: Touchable<bool>,
+    pub y_button: Touchable<bool>,
+    pub menu_button: Action<bool>,
+    pub a_button: Touchable<bool>,
+    pub b_button: Touchable<bool>,
+    pub thumbstick_x: Action<f32>,
+    pub thumbstick_y: Action<f32>,
+    pub thumbstick_touch: Action<bool>,
+    pub thumbstick_click: Action<bool>,
+    pub thumbrest_touch: Action<bool>,
+}
+impl OculusController {
+    pub fn new(
+        instance: Instance,
+        session: Session<AnyGraphics>,
+        action_sets: &mut Vec<ActionSet>,
+    ) -> anyhow::Result<Self> {
+        let action_set =
+            instance.create_action_set("oculus_input", "Oculus Touch Controller Input", 0)?;
+        let left_path = instance.string_to_path("/user/hand/left").unwrap();
+        let right_path = instance.string_to_path("/user/hand/right").unwrap();
+        let hands = [left_path, right_path];
+        let grip_pose = action_set.create_action::<Posef>("hand_pose", "Hand Pose", &hands)?;
+        let aim_pose = action_set.create_action::<Posef>("pointer_pose", "Pointer Pose", &hands)?;
+
+        let this = OculusController {
+            grip_space: Handed {
+                left: grip_pose.create_space(session.clone(), right_path, Posef::IDENTITY)?,
+                right: grip_pose.create_space(session.clone(), left_path, Posef::IDENTITY)?,
+            },
+            aim_space: Handed {
+                left: aim_pose.create_space(session.clone(), right_path, Posef::IDENTITY)?,
+                right: aim_pose.create_space(session.clone(), left_path, Posef::IDENTITY)?,
+            },
+            grip_pose,
+            aim_pose,
+            squeeze: action_set.create_action("squeeze", "Grip Pull", &hands)?,
+            trigger: Touchable {
+                inner: action_set.create_action("trigger", "Trigger Pull", &hands)?,
+                touch: action_set.create_action("trigger_touched", "Trigger Touch", &hands)?,
+            },
+            haptic_feedback: action_set.create_action(
+                "haptic_feedback",
+                "Haptic Feedback",
+                &hands,
+            )?,
+            x_button: Touchable {
+                inner: action_set.create_action("x_button", "X Button", &[])?,
+                touch: action_set.create_action("x_button_touch", "X Button Touch", &[])?,
+            },
+            y_button: Touchable {
+                inner: action_set.create_action("y_button", "Y Button", &[])?,
+                touch: action_set.create_action("y_button_touch", "Y Button Touch", &[])?,
+            },
+            menu_button: action_set.create_action("menu_button", "Menu Button", &[])?,
+            a_button: Touchable {
+                inner: action_set.create_action("a_button", "A Button", &[])?,
+                touch: action_set.create_action("a_button_touch", "A Button Touch", &[])?,
+            },
+            b_button: Touchable {
+                inner: action_set.create_action("b_button", "B Button", &[])?,
+                touch: action_set.create_action("b_button_touch", "B Button Touch", &[])?,
+            },
+            thumbstick_x: action_set.create_action("thumbstick_x", "Thumbstick X", &hands)?,
+            thumbstick_y: action_set.create_action("thumbstick_y", "Thumbstick Y", &hands)?,
+            thumbstick_touch: action_set.create_action(
+                "thumbstick_touch",
+                "Thumbstick Touch",
+                &hands,
+            )?,
+            thumbstick_click: action_set.create_action(
+                "thumbstick_click",
+                "Thumbstick Click",
+                &hands,
+            )?,
+            thumbrest_touch: action_set.create_action(
+                "thumbrest_touch",
+                "Thumbrest Touch",
+                &hands,
+            )?,
+        };
+        let i = instance;
+        i.suggest_interaction_profile_bindings(
+            i.string_to_path("/interaction_profiles/oculus/touch_controller")?,
+            &[
+                Binding::new(
+                    &this.grip_pose,
+                    i.string_to_path("/user/hand/left/input/grip/pose")?,
+                ),
+                Binding::new(
+                    &this.grip_pose,
+                    i.string_to_path("/user/hand/right/input/grip/pose")?,
+                ),
+                Binding::new(
+                    &this.aim_pose,
+                    i.string_to_path("/user/hand/left/input/aim/pose")?,
+                ),
+                Binding::new(
+                    &this.aim_pose,
+                    i.string_to_path("/user/hand/left/input/aim/pose")?,
+                ),
+                Binding::new(
+                    &this.squeeze,
+                    i.string_to_path("/user/hand/left/input/squeeze/value")?,
+                ),
+                Binding::new(
+                    &this.squeeze,
+                    i.string_to_path("/user/hand/right/input/squeeze/value")?,
+                ),
+                Binding::new(
+                    &this.trigger.inner,
+                    i.string_to_path("/user/hand/right/input/trigger/value")?,
+                ),
+                Binding::new(
+                    &this.trigger.inner,
+                    i.string_to_path("/user/hand/left/input/trigger/value")?,
+                ),
+                Binding::new(
+                    &this.trigger.touch,
+                    i.string_to_path("/user/hand/right/input/trigger/touch")?,
+                ),
+                Binding::new(
+                    &this.trigger.touch,
+                    i.string_to_path("/user/hand/left/input/trigger/touch")?,
+                ),
+                Binding::new(
+                    &this.haptic_feedback,
+                    i.string_to_path("/user/hand/right/output/haptic")?,
+                ),
+                Binding::new(
+                    &this.haptic_feedback,
+                    i.string_to_path("/user/hand/left/output/haptic")?,
+                ),
+                Binding::new(
+                    &this.x_button.inner,
+                    i.string_to_path("/user/hand/left/input/x/click")?,
+                ),
+                Binding::new(
+                    &this.x_button.touch,
+                    i.string_to_path("/user/hand/left/input/x/touch")?,
+                ),
+                Binding::new(
+                    &this.y_button.inner,
+                    i.string_to_path("/user/hand/left/input/y/click")?,
+                ),
+                Binding::new(
+                    &this.y_button.touch,
+                    i.string_to_path("/user/hand/left/input/y/touch")?,
+                ),
+                Binding::new(
+                    &this.menu_button,
+                    i.string_to_path("/user/hand/left/input/menu/click")?,
+                ),
+                Binding::new(
+                    &this.a_button.inner,
+                    i.string_to_path("/user/hand/right/input/a/click")?,
+                ),
+                Binding::new(
+                    &this.a_button.touch,
+                    i.string_to_path("/user/hand/right/input/a/touch")?,
+                ),
+                Binding::new(
+                    &this.b_button.inner,
+                    i.string_to_path("/user/hand/right/input/b/click")?,
+                ),
+                Binding::new(
+                    &this.b_button.touch,
+                    i.string_to_path("/user/hand/right/input/b/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_x,
+                    i.string_to_path("/user/hand/left/input/thumbstick/x")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_x,
+                    i.string_to_path("/user/hand/right/input/thumbstick/x")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_y,
+                    i.string_to_path("/user/hand/left/input/thumbstick/y")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_y,
+                    i.string_to_path("/user/hand/right/input/thumbstick/y")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_click,
+                    i.string_to_path("/user/hand/left/input/thumbstick/click")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_click,
+                    i.string_to_path("/user/hand/right/input/thumbstick/click")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_touch,
+                    i.string_to_path("/user/hand/left/input/thumbstick/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbstick_touch,
+                    i.string_to_path("/user/hand/right/input/thumbstick/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbrest_touch,
+                    i.string_to_path("/user/hand/left/input/thumbrest/touch")?,
+                ),
+                Binding::new(
+                    &this.thumbrest_touch,
+                    i.string_to_path("/user/hand/right/input/thumbrest/touch")?,
+                ),
+            ],
+        )?;
+
+        action_sets.push(action_set);
+        Ok(this)
+    }
+}

src/xr_input/xr_camera.rs

@@ -0,0 +1,239 @@
+use crate::xr_input::{QuatConv, Vec3Conv};
+use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE};
+use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping};
+use bevy::prelude::*;
+use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget};
+use bevy::render::primitives::Frustum;
+use bevy::render::view::{ColorGrading, VisibleEntities};
+use openxr::Fovf;
+
+#[derive(Bundle)]
+pub struct XrCamerasBundle {
+    pub left: XrCameraBundle,
+    pub right: XrCameraBundle,
+}
+impl XrCamerasBundle {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+impl Default for XrCamerasBundle {
+    fn default() -> Self {
+        Self {
+            left: XrCameraBundle::new(Eye::Left),
+            right: XrCameraBundle::new(Eye::Right),
+        }
+    }
+}
+
+#[derive(Bundle)]
+pub struct XrCameraBundle {
+    pub camera: Camera,
+    pub camera_render_graph: CameraRenderGraph,
+    pub xr_projection: XRProjection,
+    pub visible_entities: VisibleEntities,
+    pub frustum: Frustum,
+    pub transform: Transform,
+    pub global_transform: GlobalTransform,
+    pub camera_3d: Camera3d,
+    pub tonemapping: Tonemapping,
+    pub dither: DebandDither,
+    pub color_grading: ColorGrading,
+    pub xr_camera_type: XrCameraType,
+}
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)]
+pub enum XrCameraType {
+    Xr(Eye),
+    Flatscreen,
+}
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
+pub enum Eye {
+    Left = 0,
+    Right = 1,
+}
+
+impl XrCameraBundle {
+    pub fn new(eye: Eye) -> Self {
+        Self {
+            camera: Camera {
+                order: -1,
+                target: RenderTarget::TextureView(match eye {
+                    Eye::Left => LEFT_XR_TEXTURE_HANDLE,
+                    Eye::Right => RIGHT_XR_TEXTURE_HANDLE,
+                }),
+                viewport: None,
+                ..default()
+            },
+            camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME),
+            xr_projection: Default::default(),
+            visible_entities: Default::default(),
+            frustum: Default::default(),
+            transform: Default::default(),
+            global_transform: Default::default(),
+            camera_3d: Default::default(),
+            tonemapping: Default::default(),
+            dither: DebandDither::Enabled,
+            color_grading: Default::default(),
+            xr_camera_type: XrCameraType::Xr(eye),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Component, Reflect)]
+#[reflect(Component, Default)]
+pub struct XRProjection {
+    pub near: f32,
+    pub far: f32,
+    #[reflect(ignore)]
+    pub fov: Fovf,
+}
+
+impl Default for XRProjection {
+    fn default() -> Self {
+        Self {
+            near: 0.1,
+            far: 1000.,
+            fov: Default::default(),
+        }
+    }
+}
+
+impl XRProjection {
+    pub fn new(near: f32, far: f32, fov: Fovf) -> Self {
+        XRProjection { near, far, fov }
+    }
+}
+
+impl CameraProjection for XRProjection {
+    // =============================================================================
+    // math code adapted from
+    // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h
+    // Copyright (c) 2017 The Khronos Group Inc.
+    // Copyright (c) 2016 Oculus VR, LLC.
+    // SPDX-License-Identifier: Apache-2.0
+    // =============================================================================
+    fn get_projection_matrix(&self) -> Mat4 {
+        //  symmetric perspective for debugging
+        // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs());
+        // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs());
+        // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near);
+
+        let fov = self.fov;
+        let is_vulkan_api = false; // FIXME wgpu probably abstracts this
+        let near_z = self.near;
+        let far_z = -1.; //   use infinite proj
+                         // let far_z = self.far;
+
+        let tan_angle_left = fov.angle_left.tan();
+        let tan_angle_right = fov.angle_right.tan();
+
+        let tan_angle_down = fov.angle_down.tan();
+        let tan_angle_up = fov.angle_up.tan();
+
+        let tan_angle_width = tan_angle_right - tan_angle_left;
+
+        // Set to tanAngleDown - tanAngleUp for a clip space with positive Y
+        // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with
+        // positive Y up (OpenGL / D3D / Metal).
+        // const float tanAngleHeight =
+        //     graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown);
+        let tan_angle_height = if is_vulkan_api {
+            tan_angle_down - tan_angle_up
+        } else {
+            tan_angle_up - tan_angle_down
+        };
+
+        // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES).
+        // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal).
+        // const float offsetZ =
+        //     (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0;
+        // FIXME handle enum of graphics apis
+        let offset_z = 0.;
+
+        let mut cols: [f32; 16] = [0.0; 16];
+
+        if far_z <= near_z {
+            // place the far plane at infinity
+            cols[0] = 2. / tan_angle_width;
+            cols[4] = 0.;
+            cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width;
+            cols[12] = 0.;
+
+            cols[1] = 0.;
+            cols[5] = 2. / tan_angle_height;
+            cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height;
+            cols[13] = 0.;
+
+            cols[2] = 0.;
+            cols[6] = 0.;
+            cols[10] = -1.;
+            cols[14] = -(near_z + offset_z);
+
+            cols[3] = 0.;
+            cols[7] = 0.;
+            cols[11] = -1.;
+            cols[15] = 0.;
+
+            //  bevy uses the _reverse_ infinite projection
+            //  https://dev.theomader.com/depth-precision/
+            let z_reversal = Mat4::from_cols_array_2d(&[
+                [1f32, 0., 0., 0.],
+                [0., 1., 0., 0.],
+                [0., 0., -1., 0.],
+                [0., 0., 1., 1.],
+            ]);
+
+            return z_reversal * Mat4::from_cols_array(&cols);
+        } else {
+            // normal projection
+            cols[0] = 2. / tan_angle_width;
+            cols[4] = 0.;
+            cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width;
+            cols[12] = 0.;
+
+            cols[1] = 0.;
+            cols[5] = 2. / tan_angle_height;
+            cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height;
+            cols[13] = 0.;
+
+            cols[2] = 0.;
+            cols[6] = 0.;
+            cols[10] = -(far_z + offset_z) / (far_z - near_z);
+            cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z);
+
+            cols[3] = 0.;
+            cols[7] = 0.;
+            cols[11] = -1.;
+            cols[15] = 0.;
+        }
+
+        Mat4::from_cols_array(&cols)
+    }
+
+    fn update(&mut self, _width: f32, _height: f32) {}
+
+    fn far(&self) -> f32 {
+        self.far
+    }
+}
+
+pub fn xr_camera_head_sync(
+    views: ResMut<crate::resources::XrViews>,
+    mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>,
+) {
+    let mut f = || -> Option<()> {
+        for (mut transform, camera_type, mut xr_projection) in query.iter_mut() {
+            let view_idx = match camera_type {
+                XrCameraType::Xr(eye) => *eye as usize,
+                XrCameraType::Flatscreen => return None,
+            };
+            let v = views.lock().unwrap();
+            let view = v.get(view_idx)?;
+            xr_projection.fov = view.fov;
+            transform.rotation = view.pose.orientation.to_quat();
+            transform.translation = view.pose.position.to_vec3();
+        }
+        Some(())
+    };
+    let _ = f();
+}