small changes

src/lib.rs

@@ -2,11 +2,18 @@ use std::sync::Arc;
 
 use bevy::{
     app::PluginGroupBuilder,
+    core_pipeline::tonemapping::{DebandDither, Tonemapping},
+    math::Vec3A,
     prelude::*,
     render::{
-        camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews},
+        camera::{
+            CameraProjection, CameraProjectionPlugin, CameraRenderGraph, ManualTextureView,
+            ManualTextureViewHandle, ManualTextureViews, RenderTarget,
+        },
         pipelined_rendering::PipelinedRenderingPlugin,
+        primitives::Frustum,
         renderer::{render_system, RenderAdapter, RenderAdapterInfo, RenderInstance, RenderQueue},
+        view::{ColorGrading, VisibleEntities},
         Render, RenderApp, RenderPlugin,
     },
     window::PresentMode,
@@ -32,8 +39,19 @@ impl Plugin for XrPlugin {
         let (device, queue, adapter_info, adapter, instance) =
             session.get_render_resources().unwrap();
 
+        let input = session.create_input(Bindings::OculusTouch).unwrap();
+
+        let left_primary_button = input
+            .create_action(input::hand_left::PrimaryButton::CLICK)
+            .unwrap();
+
+        let left_hand_pose = input.create_action(input::hand_left::Grip::POSE).unwrap();
+
+        app.insert_non_send_resource(left_primary_button);
+        app.insert_non_send_resource(left_hand_pose);
         app.insert_non_send_resource(session.clone());
-        app.add_plugins(RenderPlugin {
+        app.add_plugins((
+            RenderPlugin {
                 render_creation: bevy::render::settings::RenderCreation::Manual(
                     device.into(),
                     RenderQueue(Arc::new(queue)),
@@ -41,34 +59,279 @@ impl Plugin for XrPlugin {
                     RenderAdapter(Arc::new(adapter)),
                     RenderInstance(Arc::new(instance)),
                 ),
-        });
+            },
+            CameraProjectionPlugin::<XRProjection>::default(),
+        ));
 
-        app.add_systems(Last, begin_frame);
+        app.add_systems(PreUpdate, begin_frame);
+        app.add_systems(Last, locate_views);
+        app.add_systems(Startup, setup);
         let render_app = app.sub_app_mut(RenderApp);
         render_app.insert_non_send_resource(session);
         render_app.add_systems(Render, end_frame.after(render_system));
     }
 }
 
-pub fn begin_frame(
+#[derive(Bundle)]
+pub struct XrCameraBundle {
+    pub camera: Camera,
+    pub camera_render_graph: CameraRenderGraph,
+    pub xr_projection: PerspectiveProjection,
+    pub visible_entities: VisibleEntities,
+    pub frustum: Frustum,
+    pub transform: Transform,
+    pub global_transform: GlobalTransform,
+    pub camera_3d: Camera3d,
+    pub tonemapping: Tonemapping,
+    pub dither: DebandDither,
+    pub color_grading: ColorGrading,
+    pub xr_camera_type: XrCameraType,
+}
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)]
+pub enum XrCameraType {
+    Xr(Eye),
+    Flatscreen,
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
+pub enum Eye {
+    Left = 0,
+    Right = 1,
+}
+
+impl XrCameraBundle {
+    pub fn new(eye: Eye) -> Self {
+        Self {
+            camera: Camera {
+                order: -1,
+                target: RenderTarget::TextureView(match eye {
+                    Eye::Left => LEFT_XR_TEXTURE_HANDLE,
+                    Eye::Right => RIGHT_XR_TEXTURE_HANDLE,
+                }),
+                viewport: None,
+                ..default()
+            },
+            camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME),
+            xr_projection: Default::default(),
+            visible_entities: Default::default(),
+            frustum: Default::default(),
+            transform: Default::default(),
+            global_transform: Default::default(),
+            camera_3d: Default::default(),
+            tonemapping: Default::default(),
+            dither: DebandDither::Enabled,
+            color_grading: Default::default(),
+            xr_camera_type: XrCameraType::Xr(eye),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Component, Reflect)]
+#[reflect(Component, Default)]
+pub struct XRProjection {
+    pub near: f32,
+    pub far: f32,
+    #[reflect(ignore)]
+    pub fov: Fov,
+}
+
+impl Default for XRProjection {
+    fn default() -> Self {
+        Self {
+            near: 0.1,
+            far: 1000.,
+            fov: Default::default(),
+        }
+    }
+}
+
+impl CameraProjection for XRProjection {
+    // =============================================================================
+    // math code adapted from
+    // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h
+    // Copyright (c) 2017 The Khronos Group Inc.
+    // Copyright (c) 2016 Oculus VR, LLC.
+    // SPDX-License-Identifier: Apache-2.0
+    // =============================================================================
+    fn get_projection_matrix(&self) -> Mat4 {
+        //  symmetric perspective for debugging
+        // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs());
+        // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs());
+        // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near);
+
+        let fov = self.fov;
+        let is_vulkan_api = false; // FIXME wgpu probably abstracts this
+        let near_z = self.near;
+        let far_z = -1.; //   use infinite proj
+                         // let far_z = self.far;
+
+        let tan_angle_left = fov.angle_left.tan();
+        let tan_angle_right = fov.angle_right.tan();
+
+        let tan_angle_down = fov.angle_down.tan();
+        let tan_angle_up = fov.angle_up.tan();
+
+        let tan_angle_width = tan_angle_right - tan_angle_left;
+
+        // Set to tanAngleDown - tanAngleUp for a clip space with positive Y
+        // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with
+        // positive Y up (OpenGL / D3D / Metal).
+        // const float tanAngleHeight =
+        //     graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown);
+        let tan_angle_height = if is_vulkan_api {
+            tan_angle_down - tan_angle_up
+        } else {
+            tan_angle_up - tan_angle_down
+        };
+
+        // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES).
+        // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal).
+        // const float offsetZ =
+        //     (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0;
+        // FIXME handle enum of graphics apis
+        let offset_z = 0.;
+
+        let mut cols: [f32; 16] = [0.0; 16];
+
+        if far_z <= near_z {
+            // place the far plane at infinity
+            cols[0] = 2. / tan_angle_width;
+            cols[4] = 0.;
+            cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width;
+            cols[12] = 0.;
+
+            cols[1] = 0.;
+            cols[5] = 2. / tan_angle_height;
+            cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height;
+            cols[13] = 0.;
+
+            cols[2] = 0.;
+            cols[6] = 0.;
+            cols[10] = -1.;
+            cols[14] = -(near_z + offset_z);
+
+            cols[3] = 0.;
+            cols[7] = 0.;
+            cols[11] = -1.;
+            cols[15] = 0.;
+
+            //  bevy uses the _reverse_ infinite projection
+            //  https://dev.theomader.com/depth-precision/
+            let z_reversal = Mat4::from_cols_array_2d(&[
+                [1f32, 0., 0., 0.],
+                [0., 1., 0., 0.],
+                [0., 0., -1., 0.],
+                [0., 0., 1., 1.],
+            ]);
+
+            return z_reversal * Mat4::from_cols_array(&cols);
+        } else {
+            // normal projection
+            cols[0] = 2. / tan_angle_width;
+            cols[4] = 0.;
+            cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width;
+            cols[12] = 0.;
+
+            cols[1] = 0.;
+            cols[5] = 2. / tan_angle_height;
+            cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height;
+            cols[13] = 0.;
+
+            cols[2] = 0.;
+            cols[6] = 0.;
+            cols[10] = -(far_z + offset_z) / (far_z - near_z);
+            cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z);
+
+            cols[3] = 0.;
+            cols[7] = 0.;
+            cols[11] = -1.;
+            cols[15] = 0.;
+        }
+
+        Mat4::from_cols_array(&cols)
+    }
+
+    fn update(&mut self, _width: f32, _height: f32) {}
+
+    fn far(&self) -> f32 {
+        self.far
+    }
+
+    fn get_frustum_corners(&self, z_near: f32, z_far: f32) -> [Vec3A; 8] {
+        let tan_angle_left = self.fov.angle_left.tan();
+        let tan_angle_right = self.fov.angle_right.tan();
+
+        let tan_angle_bottom = self.fov.angle_down.tan();
+        let tan_angle_top = self.fov.angle_up.tan();
+
+        // NOTE: These vertices are in the specific order required by [`calculate_cascade`].
+        [
+            Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_near, // bottom right
+            Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_near,    // top right
+            Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_near,     // top left
+            Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_near,  // bottom left
+            Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_far,  // bottom right
+            Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_far,     // top right
+            Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_far,      // top left
+            Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_far,   // bottom left
+        ]
+    }
+}
+
+#[derive(Resource)]
+struct Cameras(Entity, Entity);
+
+fn setup(mut commands: Commands) {
+    let left = commands.spawn(XrCameraBundle::new(Eye::Left)).id();
+    let right = commands.spawn(XrCameraBundle::new(Eye::Right)).id();
+    commands.insert_resource(Cameras(left, right));
+}
+
+pub fn begin_frame(session: NonSend<Session>, action: NonSend<Action<Pose>>) {
+    session.begin_frame().unwrap();
+}
+
+fn locate_views(
     session: NonSend<Session>,
     mut manual_texture_views: ResMut<ManualTextureViews>,
+    cameras: Res<Cameras>,
+    mut transforms: Query<(&mut Transform)>,
 ) {
-    let (left, right) = session.begin_frame().unwrap();
+    let (left_view, right_view) = session.locate_views().unwrap();
 
     let left = ManualTextureView {
-        texture_view: left.texture_view().unwrap().into(),
+        texture_view: left_view.texture_view().unwrap().into(),
-        size: left.resolution(),
+        size: left_view.resolution(),
-        format: left.format(),
+        format: left_view.format(),
     };
     let right = ManualTextureView {
-        texture_view: right.texture_view().unwrap().into(),
+        texture_view: right_view.texture_view().unwrap().into(),
-        size: right.resolution(),
+        size: right_view.resolution(),
-        format: right.format(),
+        format: right_view.format(),
     };
 
-    manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left);
+    if let Ok(mut transform) = transforms.get_mut(cameras.0) {
+        let Pose {
+            translation,
+            rotation,
+        } = left_view.pose();
+
+        transform.translation = translation;
+        transform.rotation = rotation;
+    }
+
+    if let Ok(mut transform) = transforms.get_mut(cameras.1) {
+        let Pose {
+            translation,
+            rotation,
+        } = right_view.pose();
+
+        transform.translation = translation;
+        transform.rotation = rotation;
+    }
+
     manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right);
+    manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left);
 }
 
 pub fn end_frame(session: NonSend<Session>) {

src/main.rs

@@ -45,14 +45,6 @@ fn setup(
     // camera
     commands.spawn(Camera3dBundle {
         transform: Transform::from_xyz(-2.5, 4.5, 9.0).looking_at(Vec3::ZERO, Vec3::Y),
-        camera: Camera {
-            target: RenderTarget::TextureView(LEFT_XR_TEXTURE_HANDLE),
-            ..default()
-        },
-        camera_3d: Camera3d {
-            clear_color: ClearColorConfig::Custom(Color::RED),
-            ..default()
-        },
         ..default()
     });
 }

xr_api/src/api_traits.rs

@@ -1,4 +1,4 @@
-use glam::{UVec2, Vec2};
+use glam::{UVec2, Vec2, Vec3A};
 use wgpu::{Adapter, AdapterInfo, Device, Queue, TextureView};
 
 use crate::prelude::*;
@@ -27,10 +27,14 @@ pub trait SessionTrait {
     /// Get render resources compatible with this session.
     fn get_render_resources(&self)
         -> Option<(Device, Queue, AdapterInfo, Adapter, wgpu::Instance)>;
+    /// Returns the position of the headset.
+    fn headset_location(&self) -> Result<Pose>;
     /// Request input modules with the specified bindings.
     fn create_input(&self, bindings: Bindings) -> Result<Input>;
     /// Blocks until a rendering frame is available, then returns the views for the left and right eyes.
-    fn begin_frame(&self) -> Result<(View, View)>;
+    fn begin_frame(&self) -> Result<()>;
+    /// Locate the views of each eye.
+    fn locate_views(&self) -> Result<(View, View)>;
     /// Submits rendering work for this frame.
     fn end_frame(&self) -> Result<()>;
     /// Gets the resolution of a single eye.
@@ -45,7 +49,9 @@ pub trait ViewTrait {
     /// Returns the [Pose] representing the current position of this view.
     fn pose(&self) -> Pose;
     /// Returns the projection matrix for the current view.
-    fn projection_matrix(&self) -> glam::Mat4;
+    fn projection_matrix(&self, near: f32, far: f32) -> glam::Mat4;
+    /// Gets the fov of the camera.
+    fn fov(&self) -> Fov;
     /// Gets the resolution for this view.
     fn resolution(&self) -> UVec2;
     /// Gets the texture format for the view.

xr_api/src/backend/oxr.rs

@@ -1,9 +1,11 @@
 mod graphics;
 mod utils;
 
+use utils::UntypedOXrAction;
+
 use std::{marker::PhantomData, rc::Rc, sync::Mutex};
 
-use glam::{Mat4, UVec2, Vec2};
+use glam::{Mat4, UVec2, Vec2, Vec3A};
 use hashbrown::{hash_map, HashMap};
 use openxr::{EnvironmentBlendMode, Vector2f};
 use tracing::{info, info_span, warn};
@@ -71,14 +73,6 @@ impl InstanceTrait for OXrInstance {
     }
 }
 
-pub(crate) enum UntypedOXrAction {
-    Haptics(openxr::Action<openxr::Haptic>),
-    Pose(openxr::Action<openxr::Posef>),
-    Float(openxr::Action<f32>),
-    Bool(openxr::Action<bool>),
-    Vec2(openxr::Action<openxr::Vector2f>),
-}
-
 #[derive(Default)]
 pub(crate) struct BindingState {
     sessions_attached: bool,
@@ -151,7 +145,7 @@ impl SessionTrait for OXrSession {
         .into())
     }
 
-    fn begin_frame(&self) -> Result<(View, View)> {
+    fn begin_frame(&self) -> Result<()> {
         {
             let mut bindings = self.bindings.lock().unwrap();
             if !bindings.sessions_attached {
@@ -263,6 +257,10 @@ impl SessionTrait for OXrSession {
             self.session
                 .sync_actions(&action_sets.iter().map(Into::into).collect::<Vec<_>>())?;
         }
+        Ok(())
+    }
+
+    fn locate_views(&self) -> Result<(View, View)> {
         let views = {
             let _span = info_span!("xr_locate_views").entered();
             self.session
@@ -326,6 +324,14 @@ impl SessionTrait for OXrSession {
     fn format(&self) -> wgpu::TextureFormat {
         self.format
     }
+
+    fn headset_location(&self) -> Result<Pose> {
+        let location = self.head.locate(
+            &self.stage,
+            self.frame_state.lock().unwrap().predicted_display_time,
+        )?;
+        Ok(location.pose.into())
+    }
 }
 
 pub struct OXrInput {
@@ -498,7 +504,7 @@ impl ViewTrait for OXrView {
         self.view.pose.clone().into()
     }
 
-    fn projection_matrix(&self) -> glam::Mat4 {
+    fn projection_matrix(&self, near: f32, far: f32) -> glam::Mat4 {
         //  symmetric perspective for debugging
         // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs());
         // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs());
@@ -506,9 +512,6 @@ impl ViewTrait for OXrView {
 
         let fov = self.view.fov;
         let is_vulkan_api = false; // FIXME wgpu probably abstracts this
-        let near_z = 0.1;
-        let far_z = -1.; //   use infinite proj
-                         // let far_z = self.far;
 
         let tan_angle_left = fov.angle_left.tan();
         let tan_angle_right = fov.angle_right.tan();
@@ -538,7 +541,7 @@ impl ViewTrait for OXrView {
 
         let mut cols: [f32; 16] = [0.0; 16];
 
-        if far_z <= near_z {
+        if far <= near {
             // place the far plane at infinity
             cols[0] = 2. / tan_angle_width;
             cols[4] = 0.;
@@ -553,7 +556,7 @@ impl ViewTrait for OXrView {
             cols[2] = 0.;
             cols[6] = 0.;
             cols[10] = -1.;
-            cols[14] = -(near_z + offset_z);
+            cols[14] = -(near + offset_z);
 
             cols[3] = 0.;
             cols[7] = 0.;
@@ -584,8 +587,8 @@ impl ViewTrait for OXrView {
 
             cols[2] = 0.;
             cols[6] = 0.;
-            cols[10] = -(far_z + offset_z) / (far_z - near_z);
+            cols[10] = -(far + offset_z) / (far - near);
-            cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z);
+            cols[14] = -(far * (near + offset_z)) / (far - near);
 
             cols[3] = 0.;
             cols[7] = 0.;
@@ -603,4 +606,8 @@ impl ViewTrait for OXrView {
     fn format(&self) -> wgpu::TextureFormat {
         self.format
     }
+
+    fn fov(&self) -> Fov {
+        self.view.fov.into()
+    }
 }

xr_api/src/backend/oxr/utils.rs

@@ -1,5 +1,5 @@
 use glam::Quat;
-use openxr::Posef;
+use openxr::{Action, Fovf, Posef};
 
 use crate::{
     error::XrError,
@@ -7,7 +7,7 @@ use crate::{
     prelude::Pose,
 };
 
-use super::Bindings;
+use super::{Bindings, Fov};
 
 impl From<openxr::sys::Result> for XrError {
     fn from(_: openxr::sys::Result) -> Self {
@@ -20,9 +20,9 @@ impl From<Posef> for Pose {
         // with enough sign errors anything is possible
         let rotation = {
             let o = pose.orientation;
-            Quat::from_rotation_x(180.0f32.to_radians()) * glam::quat(o.w, o.z, o.y, o.x)
+            Quat::from_xyzw(o.x, o.y, o.z, o.w)
         };
-        let translation = glam::vec3(-pose.position.x, pose.position.y, -pose.position.z);
+        let translation = glam::vec3(pose.position.x, pose.position.y, pose.position.z);
 
         Pose {
             translation,
@@ -31,6 +31,69 @@ impl From<Posef> for Pose {
     }
 }
 
+impl From<Fovf> for Fov {
+    fn from(fov: Fovf) -> Self {
+        let Fovf {
+            angle_left,
+            angle_right,
+            angle_down,
+            angle_up,
+        } = fov;
+        Self {
+            angle_down,
+            angle_left,
+            angle_right,
+            angle_up,
+        }
+    }
+}
+
+macro_rules! untyped_oxr_actions {
+    (
+        $id:ident {
+            $(
+                $inner:ident($inner_ty:ty)
+            ),*
+            $(,)?
+        }
+    ) => {
+        pub(crate) enum $id {
+            $(
+                $inner($inner_ty),
+            )*
+        }
+
+        $(
+            impl TryInto<$inner_ty> for $id {
+                type Error = ();
+
+                fn try_into(self) -> std::prelude::v1::Result<$inner_ty, Self::Error> {
+                    match self {
+                        Self::$inner(action) => Ok(action),
+                        _ => Err(()),
+                    }
+                }
+            }
+
+            impl From<$inner_ty> for $id {
+                fn from(value: $inner_ty) -> Self {
+                    Self::$inner(value)
+                }
+            }
+        )*
+    };
+}
+
+untyped_oxr_actions! {
+    UntypedOXrAction {
+        Haptics(Action<openxr::Haptic>),
+        Pose(Action<openxr::Posef>),
+        Float(Action<f32>),
+        Bool(Action<bool>),
+        Vec2(Action<openxr::Vector2f>),
+    }
+}
+
 impl UntypedActionPath {
     pub(crate) fn into_xr_path(self) -> String {
         let dev_path;

xr_api/src/types.rs

@@ -22,11 +22,20 @@ pub enum Bindings {
 
 pub struct Haptic;
 
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Pose {
     pub translation: Vec3,
     pub rotation: Quat,
 }
 
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Fov {
+    pub angle_left: f32,
+    pub angle_right: f32,
+    pub angle_down: f32,
+    pub angle_up: f32,
+}
+
 pub trait ActionType: Sized {
     type Inner: ?Sized;