diff --git a/src/lib.rs b/src/lib.rs index 92bcb85..dac2401 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,11 +2,18 @@ use std::sync::Arc; use bevy::{ app::PluginGroupBuilder, + core_pipeline::tonemapping::{DebandDither, Tonemapping}, + math::Vec3A, prelude::*, render::{ - camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}, + camera::{ + CameraProjection, CameraProjectionPlugin, CameraRenderGraph, ManualTextureView, + ManualTextureViewHandle, ManualTextureViews, RenderTarget, + }, pipelined_rendering::PipelinedRenderingPlugin, + primitives::Frustum, renderer::{render_system, RenderAdapter, RenderAdapterInfo, RenderInstance, RenderQueue}, + view::{ColorGrading, VisibleEntities}, Render, RenderApp, RenderPlugin, }, window::PresentMode, @@ -32,43 +39,299 @@ impl Plugin for XrPlugin { let (device, queue, adapter_info, adapter, instance) = session.get_render_resources().unwrap(); - app.insert_non_send_resource(session.clone()); - app.add_plugins(RenderPlugin { - render_creation: bevy::render::settings::RenderCreation::Manual( - device.into(), - RenderQueue(Arc::new(queue)), - RenderAdapterInfo(adapter_info), - RenderAdapter(Arc::new(adapter)), - RenderInstance(Arc::new(instance)), - ), - }); + let input = session.create_input(Bindings::OculusTouch).unwrap(); - app.add_systems(Last, begin_frame); + let left_primary_button = input + .create_action(input::hand_left::PrimaryButton::CLICK) + .unwrap(); + + let left_hand_pose = input.create_action(input::hand_left::Grip::POSE).unwrap(); + + app.insert_non_send_resource(left_primary_button); + app.insert_non_send_resource(left_hand_pose); + app.insert_non_send_resource(session.clone()); + app.add_plugins(( + RenderPlugin { + render_creation: bevy::render::settings::RenderCreation::Manual( + device.into(), + RenderQueue(Arc::new(queue)), + RenderAdapterInfo(adapter_info), + RenderAdapter(Arc::new(adapter)), + RenderInstance(Arc::new(instance)), + ), + }, + CameraProjectionPlugin::::default(), + )); + + app.add_systems(PreUpdate, begin_frame); + app.add_systems(Last, locate_views); + app.add_systems(Startup, setup); let render_app = app.sub_app_mut(RenderApp); render_app.insert_non_send_resource(session); render_app.add_systems(Render, end_frame.after(render_system)); } } -pub fn begin_frame( +#[derive(Bundle)] +pub struct XrCameraBundle { + pub camera: Camera, + pub camera_render_graph: CameraRenderGraph, + pub xr_projection: PerspectiveProjection, + pub visible_entities: VisibleEntities, + pub frustum: Frustum, + pub transform: Transform, + pub global_transform: GlobalTransform, + pub camera_3d: Camera3d, + pub tonemapping: Tonemapping, + pub dither: DebandDither, + pub color_grading: ColorGrading, + pub xr_camera_type: XrCameraType, +} +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] +pub enum XrCameraType { + Xr(Eye), + Flatscreen, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub enum Eye { + Left = 0, + Right = 1, +} + +impl XrCameraBundle { + pub fn new(eye: Eye) -> Self { + Self { + camera: Camera { + order: -1, + target: RenderTarget::TextureView(match eye { + Eye::Left => LEFT_XR_TEXTURE_HANDLE, + Eye::Right => RIGHT_XR_TEXTURE_HANDLE, + }), + viewport: None, + ..default() + }, + camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME), + xr_projection: Default::default(), + visible_entities: Default::default(), + frustum: Default::default(), + transform: Default::default(), + global_transform: Default::default(), + camera_3d: Default::default(), + tonemapping: Default::default(), + dither: DebandDither::Enabled, + color_grading: Default::default(), + xr_camera_type: XrCameraType::Xr(eye), + } + } +} + +#[derive(Debug, Clone, Component, Reflect)] +#[reflect(Component, Default)] +pub struct XRProjection { + pub near: f32, + pub far: f32, + #[reflect(ignore)] + pub fov: Fov, +} + +impl Default for XRProjection { + fn default() -> Self { + Self { + near: 0.1, + far: 1000., + fov: Default::default(), + } + } +} + +impl CameraProjection for XRProjection { + // ============================================================================= + // math code adapted from + // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h + // Copyright (c) 2017 The Khronos Group Inc. + // Copyright (c) 2016 Oculus VR, LLC. + // SPDX-License-Identifier: Apache-2.0 + // ============================================================================= + fn get_projection_matrix(&self) -> Mat4 { + // symmetric perspective for debugging + // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); + // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); + // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); + + let fov = self.fov; + let is_vulkan_api = false; // FIXME wgpu probably abstracts this + let near_z = self.near; + let far_z = -1.; // use infinite proj + // let far_z = self.far; + + let tan_angle_left = fov.angle_left.tan(); + let tan_angle_right = fov.angle_right.tan(); + + let tan_angle_down = fov.angle_down.tan(); + let tan_angle_up = fov.angle_up.tan(); + + let tan_angle_width = tan_angle_right - tan_angle_left; + + // Set to tanAngleDown - tanAngleUp for a clip space with positive Y + // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with + // positive Y up (OpenGL / D3D / Metal). + // const float tanAngleHeight = + // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); + let tan_angle_height = if is_vulkan_api { + tan_angle_down - tan_angle_up + } else { + tan_angle_up - tan_angle_down + }; + + // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). + // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). + // const float offsetZ = + // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; + // FIXME handle enum of graphics apis + let offset_z = 0.; + + let mut cols: [f32; 16] = [0.0; 16]; + + if far_z <= near_z { + // place the far plane at infinity + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -1.; + cols[14] = -(near_z + offset_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + + // bevy uses the _reverse_ infinite projection + // https://dev.theomader.com/depth-precision/ + let z_reversal = Mat4::from_cols_array_2d(&[ + [1f32, 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., -1., 0.], + [0., 0., 1., 1.], + ]); + + return z_reversal * Mat4::from_cols_array(&cols); + } else { + // normal projection + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -(far_z + offset_z) / (far_z - near_z); + cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + } + + Mat4::from_cols_array(&cols) + } + + fn update(&mut self, _width: f32, _height: f32) {} + + fn far(&self) -> f32 { + self.far + } + + fn get_frustum_corners(&self, z_near: f32, z_far: f32) -> [Vec3A; 8] { + let tan_angle_left = self.fov.angle_left.tan(); + let tan_angle_right = self.fov.angle_right.tan(); + + let tan_angle_bottom = self.fov.angle_down.tan(); + let tan_angle_top = self.fov.angle_up.tan(); + + // NOTE: These vertices are in the specific order required by [`calculate_cascade`]. + [ + Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_near, // bottom right + Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_near, // top right + Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_near, // top left + Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_near, // bottom left + Vec3A::new(tan_angle_right, tan_angle_bottom, 1.0) * z_far, // bottom right + Vec3A::new(tan_angle_right, tan_angle_top, 1.0) * z_far, // top right + Vec3A::new(tan_angle_left, tan_angle_top, 1.0) * z_far, // top left + Vec3A::new(tan_angle_left, tan_angle_bottom, 1.0) * z_far, // bottom left + ] + } +} + +#[derive(Resource)] +struct Cameras(Entity, Entity); + +fn setup(mut commands: Commands) { + let left = commands.spawn(XrCameraBundle::new(Eye::Left)).id(); + let right = commands.spawn(XrCameraBundle::new(Eye::Right)).id(); + commands.insert_resource(Cameras(left, right)); +} + +pub fn begin_frame(session: NonSend, action: NonSend>) { + session.begin_frame().unwrap(); +} + +fn locate_views( session: NonSend, mut manual_texture_views: ResMut, + cameras: Res, + mut transforms: Query<(&mut Transform)>, ) { - let (left, right) = session.begin_frame().unwrap(); + let (left_view, right_view) = session.locate_views().unwrap(); let left = ManualTextureView { - texture_view: left.texture_view().unwrap().into(), - size: left.resolution(), - format: left.format(), + texture_view: left_view.texture_view().unwrap().into(), + size: left_view.resolution(), + format: left_view.format(), }; let right = ManualTextureView { - texture_view: right.texture_view().unwrap().into(), - size: right.resolution(), - format: right.format(), + texture_view: right_view.texture_view().unwrap().into(), + size: right_view.resolution(), + format: right_view.format(), }; - manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); + if let Ok(mut transform) = transforms.get_mut(cameras.0) { + let Pose { + translation, + rotation, + } = left_view.pose(); + + transform.translation = translation; + transform.rotation = rotation; + } + + if let Ok(mut transform) = transforms.get_mut(cameras.1) { + let Pose { + translation, + rotation, + } = right_view.pose(); + + transform.translation = translation; + transform.rotation = rotation; + } + manual_texture_views.insert(RIGHT_XR_TEXTURE_HANDLE, right); + manual_texture_views.insert(LEFT_XR_TEXTURE_HANDLE, left); } pub fn end_frame(session: NonSend) { diff --git a/src/main.rs b/src/main.rs index 26c892e..ccbc79f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,14 +45,6 @@ fn setup( // camera commands.spawn(Camera3dBundle { transform: Transform::from_xyz(-2.5, 4.5, 9.0).looking_at(Vec3::ZERO, Vec3::Y), - camera: Camera { - target: RenderTarget::TextureView(LEFT_XR_TEXTURE_HANDLE), - ..default() - }, - camera_3d: Camera3d { - clear_color: ClearColorConfig::Custom(Color::RED), - ..default() - }, ..default() }); } diff --git a/xr_api/src/api_traits.rs b/xr_api/src/api_traits.rs index d85acb5..05c8d9d 100644 --- a/xr_api/src/api_traits.rs +++ b/xr_api/src/api_traits.rs @@ -1,4 +1,4 @@ -use glam::{UVec2, Vec2}; +use glam::{UVec2, Vec2, Vec3A}; use wgpu::{Adapter, AdapterInfo, Device, Queue, TextureView}; use crate::prelude::*; @@ -27,10 +27,14 @@ pub trait SessionTrait { /// Get render resources compatible with this session. fn get_render_resources(&self) -> Option<(Device, Queue, AdapterInfo, Adapter, wgpu::Instance)>; + /// Returns the position of the headset. + fn headset_location(&self) -> Result; /// Request input modules with the specified bindings. fn create_input(&self, bindings: Bindings) -> Result; /// Blocks until a rendering frame is available, then returns the views for the left and right eyes. - fn begin_frame(&self) -> Result<(View, View)>; + fn begin_frame(&self) -> Result<()>; + /// Locate the views of each eye. + fn locate_views(&self) -> Result<(View, View)>; /// Submits rendering work for this frame. fn end_frame(&self) -> Result<()>; /// Gets the resolution of a single eye. @@ -45,7 +49,9 @@ pub trait ViewTrait { /// Returns the [Pose] representing the current position of this view. fn pose(&self) -> Pose; /// Returns the projection matrix for the current view. - fn projection_matrix(&self) -> glam::Mat4; + fn projection_matrix(&self, near: f32, far: f32) -> glam::Mat4; + /// Gets the fov of the camera. + fn fov(&self) -> Fov; /// Gets the resolution for this view. fn resolution(&self) -> UVec2; /// Gets the texture format for the view. diff --git a/xr_api/src/backend/oxr.rs b/xr_api/src/backend/oxr.rs index bd7ce7e..3a96652 100644 --- a/xr_api/src/backend/oxr.rs +++ b/xr_api/src/backend/oxr.rs @@ -1,9 +1,11 @@ mod graphics; mod utils; +use utils::UntypedOXrAction; + use std::{marker::PhantomData, rc::Rc, sync::Mutex}; -use glam::{Mat4, UVec2, Vec2}; +use glam::{Mat4, UVec2, Vec2, Vec3A}; use hashbrown::{hash_map, HashMap}; use openxr::{EnvironmentBlendMode, Vector2f}; use tracing::{info, info_span, warn}; @@ -71,14 +73,6 @@ impl InstanceTrait for OXrInstance { } } -pub(crate) enum UntypedOXrAction { - Haptics(openxr::Action), - Pose(openxr::Action), - Float(openxr::Action), - Bool(openxr::Action), - Vec2(openxr::Action), -} - #[derive(Default)] pub(crate) struct BindingState { sessions_attached: bool, @@ -151,7 +145,7 @@ impl SessionTrait for OXrSession { .into()) } - fn begin_frame(&self) -> Result<(View, View)> { + fn begin_frame(&self) -> Result<()> { { let mut bindings = self.bindings.lock().unwrap(); if !bindings.sessions_attached { @@ -263,6 +257,10 @@ impl SessionTrait for OXrSession { self.session .sync_actions(&action_sets.iter().map(Into::into).collect::>())?; } + Ok(()) + } + + fn locate_views(&self) -> Result<(View, View)> { let views = { let _span = info_span!("xr_locate_views").entered(); self.session @@ -326,6 +324,14 @@ impl SessionTrait for OXrSession { fn format(&self) -> wgpu::TextureFormat { self.format } + + fn headset_location(&self) -> Result { + let location = self.head.locate( + &self.stage, + self.frame_state.lock().unwrap().predicted_display_time, + )?; + Ok(location.pose.into()) + } } pub struct OXrInput { @@ -498,7 +504,7 @@ impl ViewTrait for OXrView { self.view.pose.clone().into() } - fn projection_matrix(&self) -> glam::Mat4 { + fn projection_matrix(&self, near: f32, far: f32) -> glam::Mat4 { // symmetric perspective for debugging // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); @@ -506,9 +512,6 @@ impl ViewTrait for OXrView { let fov = self.view.fov; let is_vulkan_api = false; // FIXME wgpu probably abstracts this - let near_z = 0.1; - let far_z = -1.; // use infinite proj - // let far_z = self.far; let tan_angle_left = fov.angle_left.tan(); let tan_angle_right = fov.angle_right.tan(); @@ -538,7 +541,7 @@ impl ViewTrait for OXrView { let mut cols: [f32; 16] = [0.0; 16]; - if far_z <= near_z { + if far <= near { // place the far plane at infinity cols[0] = 2. / tan_angle_width; cols[4] = 0.; @@ -553,7 +556,7 @@ impl ViewTrait for OXrView { cols[2] = 0.; cols[6] = 0.; cols[10] = -1.; - cols[14] = -(near_z + offset_z); + cols[14] = -(near + offset_z); cols[3] = 0.; cols[7] = 0.; @@ -584,8 +587,8 @@ impl ViewTrait for OXrView { cols[2] = 0.; cols[6] = 0.; - cols[10] = -(far_z + offset_z) / (far_z - near_z); - cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); + cols[10] = -(far + offset_z) / (far - near); + cols[14] = -(far * (near + offset_z)) / (far - near); cols[3] = 0.; cols[7] = 0.; @@ -603,4 +606,8 @@ impl ViewTrait for OXrView { fn format(&self) -> wgpu::TextureFormat { self.format } + + fn fov(&self) -> Fov { + self.view.fov.into() + } } diff --git a/xr_api/src/backend/oxr/utils.rs b/xr_api/src/backend/oxr/utils.rs index ce1da0b..c08797d 100644 --- a/xr_api/src/backend/oxr/utils.rs +++ b/xr_api/src/backend/oxr/utils.rs @@ -1,5 +1,5 @@ use glam::Quat; -use openxr::Posef; +use openxr::{Action, Fovf, Posef}; use crate::{ error::XrError, @@ -7,7 +7,7 @@ use crate::{ prelude::Pose, }; -use super::Bindings; +use super::{Bindings, Fov}; impl From for XrError { fn from(_: openxr::sys::Result) -> Self { @@ -20,9 +20,9 @@ impl From for Pose { // with enough sign errors anything is possible let rotation = { let o = pose.orientation; - Quat::from_rotation_x(180.0f32.to_radians()) * glam::quat(o.w, o.z, o.y, o.x) + Quat::from_xyzw(o.x, o.y, o.z, o.w) }; - let translation = glam::vec3(-pose.position.x, pose.position.y, -pose.position.z); + let translation = glam::vec3(pose.position.x, pose.position.y, pose.position.z); Pose { translation, @@ -31,6 +31,69 @@ impl From for Pose { } } +impl From for Fov { + fn from(fov: Fovf) -> Self { + let Fovf { + angle_left, + angle_right, + angle_down, + angle_up, + } = fov; + Self { + angle_down, + angle_left, + angle_right, + angle_up, + } + } +} + +macro_rules! untyped_oxr_actions { + ( + $id:ident { + $( + $inner:ident($inner_ty:ty) + ),* + $(,)? + } + ) => { + pub(crate) enum $id { + $( + $inner($inner_ty), + )* + } + + $( + impl TryInto<$inner_ty> for $id { + type Error = (); + + fn try_into(self) -> std::prelude::v1::Result<$inner_ty, Self::Error> { + match self { + Self::$inner(action) => Ok(action), + _ => Err(()), + } + } + } + + impl From<$inner_ty> for $id { + fn from(value: $inner_ty) -> Self { + Self::$inner(value) + } + } + )* + }; +} + +untyped_oxr_actions! { + UntypedOXrAction { + Haptics(Action), + Pose(Action), + Float(Action), + Bool(Action), + Vec2(Action), + } +} + impl UntypedActionPath { pub(crate) fn into_xr_path(self) -> String { let dev_path; diff --git a/xr_api/src/types.rs b/xr_api/src/types.rs index 93b44c1..0b95e89 100644 --- a/xr_api/src/types.rs +++ b/xr_api/src/types.rs @@ -22,11 +22,20 @@ pub enum Bindings { pub struct Haptic; +#[derive(Clone, Copy, Debug, Default)] pub struct Pose { pub translation: Vec3, pub rotation: Quat, } +#[derive(Clone, Copy, Debug, Default)] +pub struct Fov { + pub angle_left: f32, + pub angle_right: f32, + pub angle_down: f32, + pub angle_up: f32, +} + pub trait ActionType: Sized { type Inner: ?Sized;