From 86299d31619151684932578abe5091ab85b38198 Mon Sep 17 00:00:00 2001 From: MalekiRe Date: Fri, 8 Sep 2023 22:52:29 -0700 Subject: [PATCH] input refactor --- Cargo.toml | 6 +- examples/xr.rs | 605 +++++++++++++++++------------------ src/graphics/mod.rs | 28 +- src/graphics/vulkan.rs | 117 +++---- src/input.rs | 89 +++--- src/lib.rs | 38 ++- src/resource_macros.rs | 6 +- src/resources.rs | 68 ++-- src/xr_input/controllers.rs | 14 + src/xr_input/mod.rs | 88 +++++ src/xr_input/oculus_touch.rs | 254 +++++++++++++++ src/xr_input/xr_camera.rs | 239 ++++++++++++++ 12 files changed, 1097 insertions(+), 455 deletions(-) create mode 100644 src/xr_input/controllers.rs create mode 100644 src/xr_input/mod.rs create mode 100644 src/xr_input/oculus_touch.rs create mode 100644 src/xr_input/xr_camera.rs diff --git a/Cargo.toml b/Cargo.toml index bf88e48..7f6c888 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,8 @@ edition = "2021" anyhow = "1.0.75" ash = "0.37.3" bevy = { git = "https://github.com/awtterpip/bevy", default-features = false, features = ["bevy_render"] } -openxr = { version = "0.17.1" } +openxr = { version = "0.17.1", features = ["mint"] } +mint = "0.5.9" wgpu = "0.16.0" wgpu-core = { version = "0.16.0", features = ["vulkan"] } wgpu-hal = "0.16.0" @@ -21,3 +22,6 @@ color-eyre = "0.6.2" [[example]] name = "xr" path = "examples/xr.rs" + +[profile.release] +debug = true \ No newline at end of file diff --git a/examples/xr.rs b/examples/xr.rs index ca99a3c..0e537f1 100644 --- a/examples/xr.rs +++ b/examples/xr.rs @@ -7,9 +7,9 @@ use bevy::prelude::Camera3d; use bevy::reflect::{std_traits::ReflectDefault, Reflect}; use bevy::render::view::ColorGrading; use bevy::render::{ - camera::{Camera, CameraProjection, CameraRenderGraph}, - primitives::Frustum, - view::VisibleEntities, + camera::{Camera, CameraProjection, CameraRenderGraph}, + primitives::Frustum, + view::VisibleEntities, }; use bevy::transform::components::{GlobalTransform, Transform}; // mostly copied from https://github.com/blaind/bevy_openxr/tree/main/crates/bevy_openxr/src/render_graph/camera @@ -17,174 +17,174 @@ use openxr::Fovf; #[derive(Bundle)] pub struct XrCameraBundle { - pub camera: Camera, - pub camera_render_graph: CameraRenderGraph, - pub xr_projection: XRProjection, - pub visible_entities: VisibleEntities, - pub frustum: Frustum, - pub transform: Transform, - pub global_transform: GlobalTransform, - pub camera_3d: Camera3d, - pub tonemapping: Tonemapping, - pub dither: DebandDither, - pub color_grading: ColorGrading, + pub camera: Camera, + pub camera_render_graph: CameraRenderGraph, + pub xr_projection: XRProjection, + pub visible_entities: VisibleEntities, + pub frustum: Frustum, + pub transform: Transform, + pub global_transform: GlobalTransform, + pub camera_3d: Camera3d, + pub tonemapping: Tonemapping, + pub dither: DebandDither, + pub color_grading: ColorGrading, } // NOTE: ideally Perspective and Orthographic defaults can share the same impl, but sadly it breaks rust's type inference impl Default for XrCameraBundle { - fn default() -> Self { - Self { - camera_render_graph: CameraRenderGraph::new(core_3d::graph::NAME), - camera: Default::default(), - xr_projection: Default::default(), - visible_entities: Default::default(), - frustum: Default::default(), - transform: Default::default(), - global_transform: Default::default(), - camera_3d: Default::default(), - tonemapping: Default::default(), - dither: DebandDither::Enabled, - color_grading: ColorGrading::default(), - } - } + fn default() -> Self { + Self { + camera_render_graph: CameraRenderGraph::new(core_3d::graph::NAME), + camera: Default::default(), + xr_projection: Default::default(), + visible_entities: Default::default(), + frustum: Default::default(), + transform: Default::default(), + global_transform: Default::default(), + camera_3d: Default::default(), + tonemapping: Default::default(), + dither: DebandDither::Enabled, + color_grading: ColorGrading::default(), + } + } } #[derive(Debug, Clone, Component, Reflect)] #[reflect(Component, Default)] pub struct XRProjection { - pub near: f32, - pub far: f32, - #[reflect(ignore)] - pub fov: Fovf, + pub near: f32, + pub far: f32, + #[reflect(ignore)] + pub fov: Fovf, } impl Default for XRProjection { - fn default() -> Self { - Self { - near: 0.1, - far: 1000., - fov: Default::default(), - } - } + fn default() -> Self { + Self { + near: 0.1, + far: 1000., + fov: Default::default(), + } + } } impl XRProjection { - pub fn new(near: f32, far: f32, fov: Fovf) -> Self { - XRProjection { near, far, fov } - } + pub fn new(near: f32, far: f32, fov: Fovf) -> Self { + XRProjection { near, far, fov } + } } impl CameraProjection for XRProjection { - // ============================================================================= - // math code adapted from - // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h - // Copyright (c) 2017 The Khronos Group Inc. - // Copyright (c) 2016 Oculus VR, LLC. - // SPDX-License-Identifier: Apache-2.0 - // ============================================================================= - fn get_projection_matrix(&self) -> Mat4 { - // symmetric perspective for debugging - // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); - // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); - // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); + // ============================================================================= + // math code adapted from + // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h + // Copyright (c) 2017 The Khronos Group Inc. + // Copyright (c) 2016 Oculus VR, LLC. + // SPDX-License-Identifier: Apache-2.0 + // ============================================================================= + fn get_projection_matrix(&self) -> Mat4 { + // symmetric perspective for debugging + // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); + // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); + // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); - let fov = self.fov; - let is_vulkan_api = false; // FIXME wgpu probably abstracts this - let near_z = self.near; - let far_z = -1.; // use infinite proj - // let far_z = self.far; + let fov = self.fov; + let is_vulkan_api = false; // FIXME wgpu probably abstracts this + let near_z = self.near; + let far_z = -1.; // use infinite proj + // let far_z = self.far; - let tan_angle_left = fov.angle_left.tan(); - let tan_angle_right = fov.angle_right.tan(); + let tan_angle_left = fov.angle_left.tan(); + let tan_angle_right = fov.angle_right.tan(); - let tan_angle_down = fov.angle_down.tan(); - let tan_angle_up = fov.angle_up.tan(); + let tan_angle_down = fov.angle_down.tan(); + let tan_angle_up = fov.angle_up.tan(); - let tan_angle_width = tan_angle_right - tan_angle_left; + let tan_angle_width = tan_angle_right - tan_angle_left; - // Set to tanAngleDown - tanAngleUp for a clip space with positive Y - // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with - // positive Y up (OpenGL / D3D / Metal). - // const float tanAngleHeight = - // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); - let tan_angle_height = if is_vulkan_api { - tan_angle_down - tan_angle_up - } else { - tan_angle_up - tan_angle_down - }; + // Set to tanAngleDown - tanAngleUp for a clip space with positive Y + // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with + // positive Y up (OpenGL / D3D / Metal). + // const float tanAngleHeight = + // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); + let tan_angle_height = if is_vulkan_api { + tan_angle_down - tan_angle_up + } else { + tan_angle_up - tan_angle_down + }; - // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). - // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). - // const float offsetZ = - // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; - // FIXME handle enum of graphics apis - let offset_z = 0.; + // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). + // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). + // const float offsetZ = + // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; + // FIXME handle enum of graphics apis + let offset_z = 0.; - let mut cols: [f32; 16] = [0.0; 16]; + let mut cols: [f32; 16] = [0.0; 16]; - if far_z <= near_z { - // place the far plane at infinity - cols[0] = 2. / tan_angle_width; - cols[4] = 0.; - cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; - cols[12] = 0.; + if far_z <= near_z { + // place the far plane at infinity + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; - cols[1] = 0.; - cols[5] = 2. / tan_angle_height; - cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; - cols[13] = 0.; + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; - cols[2] = 0.; - cols[6] = 0.; - cols[10] = -1.; - cols[14] = -(near_z + offset_z); + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -1.; + cols[14] = -(near_z + offset_z); - cols[3] = 0.; - cols[7] = 0.; - cols[11] = -1.; - cols[15] = 0.; + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; - // bevy uses the _reverse_ infinite projection - // https://dev.theomader.com/depth-precision/ - let z_reversal = Mat4::from_cols_array_2d(&[ - [1f32, 0., 0., 0.], - [0., 1., 0., 0.], - [0., 0., -1., 0.], - [0., 0., 1., 1.], - ]); + // bevy uses the _reverse_ infinite projection + // https://dev.theomader.com/depth-precision/ + let z_reversal = Mat4::from_cols_array_2d(&[ + [1f32, 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., -1., 0.], + [0., 0., 1., 1.], + ]); - return z_reversal * Mat4::from_cols_array(&cols); - } else { - // normal projection - cols[0] = 2. / tan_angle_width; - cols[4] = 0.; - cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; - cols[12] = 0.; + return z_reversal * Mat4::from_cols_array(&cols); + } else { + // normal projection + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; - cols[1] = 0.; - cols[5] = 2. / tan_angle_height; - cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; - cols[13] = 0.; + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; - cols[2] = 0.; - cols[6] = 0.; - cols[10] = -(far_z + offset_z) / (far_z - near_z); - cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -(far_z + offset_z) / (far_z - near_z); + cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); - cols[3] = 0.; - cols[7] = 0.; - cols[11] = -1.; - cols[15] = 0.; - } + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + } - Mat4::from_cols_array(&cols) - } + Mat4::from_cols_array(&cols) + } - fn update(&mut self, _width: f32, _height: f32) {} + fn update(&mut self, _width: f32, _height: f32) {} - fn far(&self) -> f32 { - self.far - } + fn far(&self) -> f32 { + self.far + } } use bevy::render::camera::CameraProjectionPlugin; @@ -193,210 +193,203 @@ use bevy::transform::TransformSystem; use bevy::{prelude::*, render::camera::RenderTarget}; use bevy_openxr::input::XrInput; use bevy_openxr::resources::{XrFrameState, XrSession, XrViews}; +use bevy_openxr::xr_input::controllers::XrControllerType; +use bevy_openxr::xr_input::oculus_touch::OculusController; +use bevy_openxr::xr_input::OpenXrInput; use bevy_openxr::{DefaultXrPlugins, LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; use openxr::ActiveActionSet; fn main() { - color_eyre::install().unwrap(); + color_eyre::install().unwrap(); - info!("Running `openxr-6dof` skill"); - App::new() - .add_plugins(DefaultXrPlugins) - .add_plugins(CameraProjectionPlugin::::default()) - .add_systems(Startup, setup) - .add_systems(PreUpdate, head_movement) - .add_systems(PreUpdate, hands) - .add_systems( - PostUpdate, - update_frusta:: - .after(TransformSystem::TransformPropagate) - .before(VisibilitySystems::UpdatePerspectiveFrusta), - ) - .run(); + info!("Running `openxr-6dof` skill"); + App::new() + .add_plugins(DefaultXrPlugins) + .add_plugins(OpenXrInput::new(XrControllerType::OculusTouch)) + .add_systems(Startup, setup) + .add_systems(Update, hands) + .run(); } #[derive(Component)] enum CameraType { - Left, - Right, - Middle, + Left, + Right, + Middle, } /// set up a simple 3D scene fn setup( - mut commands: Commands, - mut meshes: ResMut>, - mut materials: ResMut>, + mut commands: Commands, + mut meshes: ResMut>, + mut materials: ResMut>, ) { - // plane - commands.spawn(PbrBundle { - mesh: meshes.add(shape::Plane::from_size(5.0).into()), - material: materials.add(Color::rgb(0.3, 0.5, 0.3).into()), - ..default() - }); - // cube - commands.spawn(PbrBundle { - mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })), - material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()), - transform: Transform::from_xyz(0.0, 0.5, 0.0), - ..default() - }); - // light - commands.spawn(PointLightBundle { - point_light: PointLight { - intensity: 1500.0, - shadows_enabled: true, - ..default() - }, - transform: Transform::from_xyz(4.0, 8.0, 4.0), - ..default() - }); - // camera - commands.spawn(( - Camera3dBundle { - transform: Transform::from_xyz(-2.0, 2.5, 5.0) - .looking_at(Vec3::ZERO, Vec3::Y), - ..default() - }, - CameraType::Middle, - )); - - commands.spawn(( - XrCameraBundle { - transform: Transform::from_xyz(-2.0, 2.5, 5.0) - .looking_at(Vec3::ZERO, Vec3::Y), - camera: Camera { - order: -1, - target: RenderTarget::TextureView(LEFT_XR_TEXTURE_HANDLE), - viewport: None, - ..default() - }, - ..default() - }, - CameraType::Left, - )); - commands.spawn(( - XrCameraBundle { - transform: Transform::from_xyz(-2.0, 2.5, 5.0) - .looking_at(Vec3::ZERO, Vec3::Y), - camera: Camera { - order: -1, - target: RenderTarget::TextureView(RIGHT_XR_TEXTURE_HANDLE), - viewport: None, - ..default() - }, - ..default() - }, - CameraType::Right, - )); + // plane + commands.spawn(PbrBundle { + mesh: meshes.add(shape::Plane::from_size(5.0).into()), + material: materials.add(Color::rgb(0.3, 0.5, 0.3).into()), + ..default() + }); + // cube + commands.spawn(PbrBundle { + mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })), + material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()), + transform: Transform::from_xyz(0.0, 0.5, 0.0), + ..default() + }); + // light + commands.spawn(PointLightBundle { + point_light: PointLight { + intensity: 1500.0, + shadows_enabled: true, + ..default() + }, + transform: Transform::from_xyz(4.0, 8.0, 4.0), + ..default() + }); + // camera + commands.spawn(( + Camera3dBundle { + transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y), + ..default() + }, + CameraType::Middle, + )); } fn hands( - mut gizmos: Gizmos, - xr_input: Res, - session: Res, - frame_state: Res, + mut gizmos: Gizmos, + oculus_controller: Res, + frame_state: Res, + xr_input: Res, ) { - //let pose = xr_input.left_action.create_space(Session::clone(&session), Path, Posef::IDENTITY).unwrap(); - let act = ActiveActionSet::new(&xr_input.action_set); - session.sync_actions(&[act]).unwrap(); - frame_state.lock().unwrap().map(|a| { - //let b = pose.locate(&*xr_input.stage, a.predicted_display_time).unwrap(); - let b = xr_input - .left_space - .relate(&xr_input.stage, a.predicted_display_time) - .unwrap(); - gizmos.rect( - b.0.pose.position.to_vec3(), - b.0.pose.orientation.to_quat(), - Vec2::new(0.05, 0.2), - Color::YELLOW_GREEN, - ); - let c = xr_input - .right_space - .relate(&xr_input.stage, a.predicted_display_time) - .unwrap(); - gizmos.rect( - c.0.pose.position.to_vec3(), - c.0.pose.orientation.to_quat(), - Vec2::new(0.05, 0.2), - Color::YELLOW_GREEN, - ) - }); + frame_state.lock().unwrap().map(|a| { + let right_controller = oculus_controller + .grip_space + .right + .relate(&**&xr_input.stage, a.predicted_display_time) + .unwrap(); + let left_controller = oculus_controller + .grip_space + .left + .relate(&**&xr_input.stage, a.predicted_display_time) + .unwrap(); + gizmos.rect( + right_controller.0.pose.position.to_vec3(), + right_controller.0.pose.orientation.to_quat(), + Vec2::new(0.05, 0.2), + Color::YELLOW_GREEN, + ); + gizmos.rect( + left_controller.0.pose.position.to_vec3(), + left_controller.0.pose.orientation.to_quat(), + Vec2::new(0.05, 0.2), + Color::YELLOW_GREEN, + ); + }); } +/*fn hands( + mut gizmos: Gizmos, + xr_input: Res, + session: Res, + frame_state: Res, +) { + //let pose = xr_input.left_action.create_space(Session::clone(&session), Path, Posef::IDENTITY).unwrap(); + let act = ActiveActionSet::new(&xr_input.action_set); + session.sync_actions(&[act]).unwrap(); + frame_state.lock().unwrap().map(|a| { + //let b = pose.locate(&*xr_input.stage, a.predicted_display_time).unwrap(); + let b = xr_input + .left_space + .relate(&xr_input.stage, a.predicted_display_time) + .unwrap(); + gizmos.rect( + b.0.pose.position.to_vec3(), + b.0.pose.orientation.to_quat(), + Vec2::new(0.05, 0.2), + Color::YELLOW_GREEN, + ); + let c = xr_input + .right_space + .relate(&xr_input.stage, a.predicted_display_time) + .unwrap(); + gizmos.rect( + c.0.pose.position.to_vec3(), + c.0.pose.orientation.to_quat(), + Vec2::new(0.05, 0.2), + Color::YELLOW_GREEN, + ) + }); +}*/ + fn head_movement( - views: ResMut, - mut query: Query<(&mut Transform, &mut Camera, &CameraType, &mut XRProjection)>, + views: ResMut, + mut query: Query<(&mut Transform, &mut Camera, &CameraType, &mut XRProjection)>, ) { - let views = views.lock().unwrap(); - let mut f = || -> Option<()> { - let midpoint = (views.get(0)?.pose.position.to_vec3() - + views.get(1)?.pose.position.to_vec3()) - / 2.; - for (mut t, _, camera_type, _) in query.iter_mut() { - match camera_type { - CameraType::Left => { - t.translation = views.get(0)?.pose.position.to_vec3() - } - CameraType::Right => { - t.translation = views.get(1)?.pose.position.to_vec3() - } - CameraType::Middle => { - t.translation = midpoint; - } - } - } - let left_rot = views.get(0).unwrap().pose.orientation.to_quat(); - let right_rot = views.get(1).unwrap().pose.orientation.to_quat(); - let mid_rot = if left_rot.dot(right_rot) >= 0. { - left_rot.slerp(right_rot, 0.5) - } else { - right_rot.slerp(left_rot, 0.5) - }; - for (mut t, _, camera_type, _) in query.iter_mut() { - match camera_type { - CameraType::Left => t.rotation = left_rot, - CameraType::Right => t.rotation = right_rot, - CameraType::Middle => { - t.rotation = mid_rot; - } - } - } + let views = views.lock().unwrap(); + let mut f = || -> Option<()> { + let midpoint = + (views.get(0)?.pose.position.to_vec3() + views.get(1)?.pose.position.to_vec3()) / 2.; + for (mut t, _, camera_type, _) in query.iter_mut() { + match camera_type { + CameraType::Left => t.translation = views.get(0)?.pose.position.to_vec3(), + CameraType::Right => t.translation = views.get(1)?.pose.position.to_vec3(), + CameraType::Middle => { + t.translation = midpoint; + } + } + } + let left_rot = views.get(0).unwrap().pose.orientation.to_quat(); + let right_rot = views.get(1).unwrap().pose.orientation.to_quat(); + let mid_rot = if left_rot.dot(right_rot) >= 0. { + left_rot.slerp(right_rot, 0.5) + } else { + right_rot.slerp(left_rot, 0.5) + }; + for (mut t, _, camera_type, _) in query.iter_mut() { + match camera_type { + CameraType::Left => t.rotation = left_rot, + CameraType::Right => t.rotation = right_rot, + CameraType::Middle => { + t.rotation = mid_rot; + } + } + } - for (mut transform, _cam, camera_type, mut xr_projection) in query.iter_mut() { - let view_idx = match camera_type { - CameraType::Left => 0, - CameraType::Right => 1, - CameraType::Middle => panic!(), - }; - let view = views.get(view_idx).unwrap(); - xr_projection.fov = view.fov; + for (mut transform, _cam, camera_type, mut xr_projection) in query.iter_mut() { + let view_idx = match camera_type { + CameraType::Left => 0, + CameraType::Right => 1, + CameraType::Middle => panic!(), + }; + let view = views.get(view_idx).unwrap(); + xr_projection.fov = view.fov; - transform.rotation = view.pose.orientation.to_quat(); - let pos = view.pose.position; - transform.translation = pos.to_vec3(); - } + transform.rotation = view.pose.orientation.to_quat(); + let pos = view.pose.position; + transform.translation = pos.to_vec3(); + } - Some(()) - }; - f(); + Some(()) + }; + f(); } pub trait Vec3Conv { - fn to_vec3(&self) -> Vec3; + fn to_vec3(&self) -> Vec3; } impl Vec3Conv for openxr::Vector3f { - fn to_vec3(&self) -> Vec3 { - Vec3::new(self.x, self.y, self.z) - } + fn to_vec3(&self) -> Vec3 { + Vec3::new(self.x, self.y, self.z) + } } pub trait QuatConv { - fn to_quat(&self) -> Quat; + fn to_quat(&self) -> Quat; } impl QuatConv for openxr::Quaternionf { - fn to_quat(&self) -> Quat { - Quat::from_xyzw(self.x, self.y, self.z, self.w) - } -} \ No newline at end of file + fn to_quat(&self) -> Quat { + Quat::from_xyzw(self.x, self.y, self.z, self.w) + } +} diff --git a/src/graphics/mod.rs b/src/graphics/mod.rs index 5d1deed..cc9cedd 100644 --- a/src/graphics/mod.rs +++ b/src/graphics/mod.rs @@ -1,12 +1,32 @@ mod vulkan; -use bevy::render::renderer::{RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter}; +use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}; use bevy::window::RawHandleWrapper; use wgpu::Instance; use crate::input::XrInput; -use crate::resources::{XrInstance, XrSession, XrEnvironmentBlendMode, XrSessionRunning, XrFrameWaiter, XrSwapchain, XrViews, XrFrameState}; +use crate::resources::{ + XrEnvironmentBlendMode, XrFrameState, XrFrameWaiter, XrInstance, XrSession, XrSessionRunning, + XrSwapchain, XrViews, +}; -pub fn initialize_xr_graphics(window: Option) -> anyhow::Result<(RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter, Instance, XrInstance, XrSession, XrEnvironmentBlendMode, XrSessionRunning, XrFrameWaiter, XrSwapchain, XrInput, XrViews, XrFrameState)>{ +pub fn initialize_xr_graphics( + window: Option, +) -> anyhow::Result<( + RenderDevice, + RenderQueue, + RenderAdapterInfo, + RenderAdapter, + Instance, + XrInstance, + XrSession, + XrEnvironmentBlendMode, + XrSessionRunning, + XrFrameWaiter, + XrSwapchain, + XrInput, + XrViews, + XrFrameState, +)> { vulkan::initialize_xr_graphics(window) -} \ No newline at end of file +} diff --git a/src/graphics/vulkan.rs b/src/graphics/vulkan.rs index 9b77667..bfd1cf4 100644 --- a/src/graphics/vulkan.rs +++ b/src/graphics/vulkan.rs @@ -1,6 +1,6 @@ use std::ffi::{c_void, CString}; -use std::sync::{Arc, Mutex}; use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Mutex}; use anyhow::Context; use ash::vk::{self, Handle}; @@ -13,11 +13,14 @@ use wgpu::{Instance, Texture}; use crate::input::XrInput; use crate::resources::{ - XrEnvironmentBlendMode, XrFrameWaiter, XrInstance, XrSession, XrSessionRunning, XrSwapchain, Swapchain, SwapchainInner, XrViews, XrFrameState, + Swapchain, SwapchainInner, XrEnvironmentBlendMode, XrFrameState, XrFrameWaiter, XrInstance, + XrSession, XrSessionRunning, XrSwapchain, XrViews, }; use crate::VIEW_TYPE; -pub fn initialize_xr_graphics(window: Option) -> anyhow::Result<( +pub fn initialize_xr_graphics( + window: Option, +) -> anyhow::Result<( RenderDevice, RenderQueue, RenderAdapterInfo, @@ -277,11 +280,14 @@ pub fn initialize_xr_graphics(window: Option) -> anyhow::Resul .expect("Failed to create wgpu surface") }); let swapchain_format = surface - .as_ref() - .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0]) - .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb); + .as_ref() + .map(|surface| surface.get_capabilities(&wgpu_adapter).formats[0]) + .unwrap_or(wgpu::TextureFormat::Rgba8UnormSrgb); - let resolution = uvec2(views[0].recommended_image_rect_width, views[0].recommended_image_rect_height); + let resolution = uvec2( + views[0].recommended_image_rect_width, + views[0].recommended_image_rect_height, + ); let handle = session .create_swapchain(&xr::SwapchainCreateInfo { @@ -303,54 +309,54 @@ pub fn initialize_xr_graphics(window: Option) -> anyhow::Resul let images = handle.enumerate_images().unwrap(); let buffers = images - .into_iter() - .map(|color_image| { - let color_image = vk::Image::from_raw(color_image); - let wgpu_hal_texture = unsafe { - ::Device::texture_from_raw( - color_image, - &wgpu_hal::TextureDescriptor { - label: Some("VR Swapchain"), - size: wgpu::Extent3d { - width: resolution.x, - height: resolution.y, - depth_or_array_layers: 2, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: swapchain_format, - usage: wgpu_hal::TextureUses::COLOR_TARGET - | wgpu_hal::TextureUses::COPY_DST, - memory_flags: wgpu_hal::MemoryFlags::empty(), - view_formats: vec![], + .into_iter() + .map(|color_image| { + let color_image = vk::Image::from_raw(color_image); + let wgpu_hal_texture = unsafe { + ::Device::texture_from_raw( + color_image, + &wgpu_hal::TextureDescriptor { + label: Some("VR Swapchain"), + size: wgpu::Extent3d { + width: resolution.x, + height: resolution.y, + depth_or_array_layers: 2, }, - None, - ) - }; - let texture = unsafe { - wgpu_device.create_texture_from_hal::( - wgpu_hal_texture, - &wgpu::TextureDescriptor { - label: Some("VR Swapchain"), - size: wgpu::Extent3d { - width: resolution.x, - height: resolution.y, - depth_or_array_layers: 2, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: swapchain_format, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT - | wgpu::TextureUsages::COPY_DST, - view_formats: &[], + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: swapchain_format, + usage: wgpu_hal::TextureUses::COLOR_TARGET + | wgpu_hal::TextureUses::COPY_DST, + memory_flags: wgpu_hal::MemoryFlags::empty(), + view_formats: vec![], + }, + None, + ) + }; + let texture = unsafe { + wgpu_device.create_texture_from_hal::( + wgpu_hal_texture, + &wgpu::TextureDescriptor { + label: Some("VR Swapchain"), + size: wgpu::Extent3d { + width: resolution.x, + height: resolution.y, + depth_or_array_layers: 2, }, - ) - }; - texture - }) - .collect(); + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: swapchain_format, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }, + ) + }; + texture + }) + .collect(); Ok(( wgpu_device.into(), @@ -370,7 +376,8 @@ pub fn initialize_xr_graphics(window: Option) -> anyhow::Resul format: swapchain_format, buffers, image_index: 0, - })).into(), + })) + .into(), XrInput::new(xr_instance, session.into_any_graphics())?, Mutex::default().into(), Mutex::default().into(), @@ -441,4 +448,4 @@ fn wgpu_to_vulkan(format: wgpu::TextureFormat) -> vk::Format { wgpu::TextureFormat::Astc { .. } => panic!("please god kill me now"), _ => panic!("fuck no") } -} \ No newline at end of file +} diff --git a/src/input.rs b/src/input.rs index 1a1564a..54f43bb 100644 --- a/src/input.rs +++ b/src/input.rs @@ -7,57 +7,58 @@ type XrPose = (Vec3, Quat); #[derive(Clone, Resource)] pub struct XrInput { - pub action_set: xr::ActionSet, - pub hand_pose: xr::Action, - pub right_space: Arc, - pub left_space: Arc, + //pub action_set: xr::ActionSet, + //pub hand_pose: xr::Action, + //pub right_space: Arc, + //pub left_space: Arc, pub stage: Arc, } impl XrInput { - pub fn new( - instance: xr::Instance, - session: xr::Session, - ) -> xr::Result { - let action_set = instance.create_action_set("input", "input pose information", 0)?; - let left_hand_subaction_path = instance.string_to_path("/user/hand/left").unwrap(); - let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap(); - let left_hand_grip_pose_path = instance - .string_to_path("/user/hand/left/input/grip/pose") - .unwrap(); - let right_hand_grip_pose_path = instance - .string_to_path("/user/hand/right/input/grip/pose") - .unwrap(); - let hand_pose = - action_set.create_action::("hand_pose", "Hand Pose", &[left_hand_subaction_path, right_hand_subaction_path])?; - /* let left_action = - action_set.create_action::("left_hand", "Left Hand Controller", &[])?;*/ - instance.suggest_interaction_profile_bindings( - instance.string_to_path("/interaction_profiles/khr/simple_controller")?, - &[ - xr::Binding::new( - &hand_pose, - right_hand_grip_pose_path, - ), - xr::Binding::new( - &hand_pose, - left_hand_grip_pose_path, - ), - ], - )?; - - let right_space = - hand_pose.create_space(session.clone(), right_hand_subaction_path, xr::Posef::IDENTITY)?; - let left_space = - hand_pose.create_space(session.clone(), left_hand_subaction_path, xr::Posef::IDENTITY)?; + pub fn new(instance: xr::Instance, session: xr::Session) -> xr::Result { + // let action_set = instance.create_action_set("input", "input pose information", 0)?; + // let left_hand_subaction_path = instance.string_to_path("/user/hand/left").unwrap(); + // let right_hand_subaction_path = instance.string_to_path("/user/hand/right").unwrap(); + // let left_hand_grip_pose_path = instance + // .string_to_path("/user/hand/left/input/grip/pose") + // .unwrap(); + // let right_hand_grip_pose_path = instance + // .string_to_path("/user/hand/right/input/grip/pose") + // .unwrap(); + // let hand_pose = action_set.create_action::( + // "hand_pose", + // "Hand Pose", + // &[left_hand_subaction_path, right_hand_subaction_path], + // )?; + // /* let left_action = + // action_set.create_action::("left_hand", "Left Hand Controller", &[])?;*/ + // instance.suggest_interaction_profile_bindings( + // instance.string_to_path("/interaction_profiles/khr/simple_controller")?, + // &[ + // xr::Binding::new(&hand_pose, right_hand_grip_pose_path), + // xr::Binding::new(&hand_pose, left_hand_grip_pose_path), + // ], + // )?; + // + // let right_space = hand_pose.create_space( + // session.clone(), + // right_hand_subaction_path, + // xr::Posef::IDENTITY, + // )?; + // let left_space = hand_pose.create_space( + // session.clone(), + // left_hand_subaction_path, + // xr::Posef::IDENTITY, + // )?; let stage = session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?; - session.attach_action_sets(&[&action_set])?; + //session.attach_action_sets(&[&action_set])?; + //session.attach_action_sets(&[])?; Ok(Self { - action_set, - hand_pose, - right_space: Arc::new(right_space), - left_space: Arc::new(left_space), + //action_set, + //hand_pose, + // right_space: Arc::new(right_space), + // left_space: Arc::new(left_space), stage: Arc::new(stage), }) } diff --git a/src/lib.rs b/src/lib.rs index ddbd364..424fb85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,14 +2,16 @@ mod graphics; pub mod input; pub mod resource_macros; pub mod resources; +pub mod xr_input; use std::sync::{Arc, Mutex}; +use crate::xr_input::oculus_touch::ActionSets; use bevy::app::PluginGroupBuilder; use bevy::ecs::system::SystemState; use bevy::prelude::*; use bevy::render::camera::{ManualTextureView, ManualTextureViewHandle, ManualTextureViews}; -use bevy::render::renderer::{RenderAdapterInfo, RenderAdapter, RenderDevice, RenderQueue}; +use bevy::render::renderer::{RenderAdapter, RenderAdapterInfo, RenderDevice, RenderQueue}; use bevy::render::settings::RenderSettings; use bevy::render::{Render, RenderApp, RenderPlugin, RenderSet}; use bevy::window::{PrimaryWindow, RawHandleWrapper}; @@ -85,7 +87,16 @@ impl Plugin for OpenXrPlugin { views, frame_state, )); - app.add_plugins(DefaultPlugins.set(RenderPlugin { render_settings: RenderSettings::Manual(device, queue, adapter_info, render_adapter, Mutex::new(instance))})); + app.insert_resource(ActionSets(vec![])); + app.add_plugins(DefaultPlugins.set(RenderPlugin { + render_settings: RenderSettings::Manual( + device, + queue, + adapter_info, + render_adapter, + Mutex::new(instance), + ), + })); } fn ready(&self, app: &App) -> bool { @@ -109,6 +120,8 @@ impl Plugin for OpenXrPlugin { frame_state, ) = future_renderer_resources.0.lock().unwrap().take().unwrap(); + let action_sets = app.world.resource::().clone(); + app.insert_resource(xr_instance.clone()) .insert_resource(session.clone()) .insert_resource(blend_mode.clone()) @@ -117,7 +130,8 @@ impl Plugin for OpenXrPlugin { .insert_resource(swapchain.clone()) .insert_resource(input.clone()) .insert_resource(views.clone()) - .insert_resource(frame_state.clone()); + .insert_resource(frame_state.clone()) + .insert_resource(action_sets.clone()); let swapchain_mut = swapchain.lock().unwrap(); let (left, right) = swapchain_mut.get_render_views(); @@ -148,7 +162,8 @@ impl Plugin for OpenXrPlugin { .insert_resource(swapchain) .insert_resource(input) .insert_resource(views) - .insert_resource(frame_state); + .insert_resource(frame_state) + .insert_resource(action_sets); render_app.add_systems( Render, @@ -180,6 +195,7 @@ pub fn pre_frame( frame_waiter: Res, swapchain: Res, xr_input: Res, + action_sets: Res, mut manual_texture_views: ResMut, ) { while let Some(event) = instance.poll_event(&mut Default::default()).unwrap() { @@ -211,7 +227,7 @@ pub fn pre_frame( } if !session_running.load(std::sync::atomic::Ordering::Relaxed) { // Don't grind up the CPU - std::thread::sleep(std::time::Duration::from_millis(10)); + //std::thread::sleep(std::time::Duration::from_millis(10)); return; } @@ -222,13 +238,17 @@ pub fn pre_frame( swapchain.begin().unwrap(); swapchain.update_render_views(); let (left, right) = swapchain.get_render_views(); - let active_action_set = xr::ActiveActionSet::new(&xr_input.action_set); - match session.sync_actions(&[active_action_set]) { + /*let mut active_action_sets = vec![]; + for i in &action_sets.0 { + active_action_sets.push(xr::ActiveActionSet::new(i)); + } + info!("action sets: {:#?}", action_sets.0.len()); + match session.sync_actions(&active_action_sets) { Err(err) => { - eprintln!("{}", err); + warn!("{}", err); } _ => {} - } + }*/ let format = swapchain.format(); let left = ManualTextureView { texture_view: left.into(), diff --git a/src/resource_macros.rs b/src/resource_macros.rs index 358b6ee..5f4df4e 100644 --- a/src/resource_macros.rs +++ b/src/resource_macros.rs @@ -23,7 +23,7 @@ macro_rules! xr_resource_wrapper { Self::new(value) } } - } + }; } #[macro_export] @@ -51,8 +51,8 @@ macro_rules! xr_arc_resource_wrapper { Self::new(value) } } - } + }; } +pub use xr_arc_resource_wrapper; pub use xr_resource_wrapper; -pub use xr_arc_resource_wrapper; \ No newline at end of file diff --git a/src/resources.rs b/src/resources.rs index cc8185e..371ff45 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -40,7 +40,7 @@ impl Swapchain { pub(crate) fn format(&self) -> wgpu::TextureFormat { match self { - Swapchain::Vulkan(swap) => swap.format + Swapchain::Vulkan(swap) => swap.format, } } @@ -58,7 +58,9 @@ impl Swapchain { environment_blend_mode: xr::EnvironmentBlendMode, ) -> xr::Result<()> { match self { - Swapchain::Vulkan(swap) => swap.post_queue_submit(xr_frame_state, views, stage, environment_blend_mode), + Swapchain::Vulkan(swap) => { + swap.post_queue_submit(xr_frame_state, views, stage, environment_blend_mode) + } } } } @@ -110,36 +112,36 @@ impl SwapchainInner { environment_blend_mode: xr::EnvironmentBlendMode, ) -> xr::Result<()> { self.handle.release_image().unwrap(); - let rect = xr::Rect2Di { - offset: xr::Offset2Di { x: 0, y: 0 }, - extent: xr::Extent2Di { - width: self.resolution.x as _, - height: self.resolution.y as _, - }, - }; - self.stream.end( - xr_frame_state.predicted_display_time, - environment_blend_mode, - &[&xr::CompositionLayerProjection::new().space(stage).views(&[ - xr::CompositionLayerProjectionView::new() - .pose(views[0].pose) - .fov(views[0].fov) - .sub_image( - xr::SwapchainSubImage::new() - .swapchain(&self.handle) - .image_array_index(0) - .image_rect(rect), - ), - xr::CompositionLayerProjectionView::new() - .pose(views[1].pose) - .fov(views[1].fov) - .sub_image( - xr::SwapchainSubImage::new() - .swapchain(&self.handle) - .image_array_index(1) - .image_rect(rect), - ), - ])], - ) + let rect = xr::Rect2Di { + offset: xr::Offset2Di { x: 0, y: 0 }, + extent: xr::Extent2Di { + width: self.resolution.x as _, + height: self.resolution.y as _, + }, + }; + self.stream.end( + xr_frame_state.predicted_display_time, + environment_blend_mode, + &[&xr::CompositionLayerProjection::new().space(stage).views(&[ + xr::CompositionLayerProjectionView::new() + .pose(views[0].pose) + .fov(views[0].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&self.handle) + .image_array_index(0) + .image_rect(rect), + ), + xr::CompositionLayerProjectionView::new() + .pose(views[1].pose) + .fov(views[1].fov) + .sub_image( + xr::SwapchainSubImage::new() + .swapchain(&self.handle) + .image_array_index(1) + .image_rect(rect), + ), + ])], + ) } } diff --git a/src/xr_input/controllers.rs b/src/xr_input/controllers.rs new file mode 100644 index 0000000..dd3c743 --- /dev/null +++ b/src/xr_input/controllers.rs @@ -0,0 +1,14 @@ +use openxr::{Action, ActionTy}; + +pub struct Touchable { + pub inner: Action, + pub touch: Action, +} +pub struct Handed { + pub left: T, + pub right: T, +} +#[derive(Copy, Clone)] +pub enum XrControllerType { + OculusTouch, +} diff --git a/src/xr_input/mod.rs b/src/xr_input/mod.rs new file mode 100644 index 0000000..244cffe --- /dev/null +++ b/src/xr_input/mod.rs @@ -0,0 +1,88 @@ +pub mod controllers; +pub mod oculus_touch; +pub mod xr_camera; + +use crate::resources::XrSession; +use crate::xr_input::controllers::XrControllerType; +use crate::xr_input::oculus_touch::{setup_oculus_controller, ActionSets}; +use crate::xr_input::xr_camera::{ + xr_camera_head_sync, Eye, XRProjection, XrCameraBundle, XrCamerasBundle, +}; +use bevy::app::{App, PostUpdate, Startup}; +use bevy::log::{info, warn}; +use bevy::prelude::IntoSystemConfigs; +use bevy::prelude::{ + Commands, Component, IntoSystemSetConfigs, Plugin, PreUpdate, Quat, Res, Resource, Vec3, +}; +use bevy::render::camera::CameraProjectionPlugin; +use bevy::render::view::{update_frusta, VisibilitySystems}; +use bevy::transform::TransformSystem; +use openxr::{Action, ActionSet, ActionTy}; + +#[derive(Copy, Clone)] +pub struct OpenXrInput { + pub controller_type: XrControllerType, +} +impl OpenXrInput { + pub fn new(controller_type: XrControllerType) -> Self { + Self { controller_type } + } +} + +impl Plugin for OpenXrInput { + fn build(&self, app: &mut App) { + app.add_plugins(CameraProjectionPlugin::::default()); + match self.controller_type { + XrControllerType::OculusTouch => { + app.add_systems(Startup, setup_oculus_controller); + } + } + app.add_systems(PreUpdate, action_set_system); + app.add_systems(PreUpdate, xr_camera_head_sync); + app.add_systems( + PostUpdate, + update_frusta:: + .after(TransformSystem::TransformPropagate) + .before(VisibilitySystems::UpdatePerspectiveFrusta), + ); + app.add_systems(Startup, setup_xr_cameras); + } +} + +fn setup_xr_cameras(mut commands: Commands) { + commands.spawn(XrCameraBundle::new(Eye::Right)); + commands.spawn(XrCameraBundle::new(Eye::Left)); +} + +fn action_set_system(action_sets: Res, session: Res) { + let mut active_action_sets = vec![]; + for i in &action_sets.0 { + active_action_sets.push(openxr::ActiveActionSet::new(i)); + } + info!("action sets: {:#?}", action_sets.0.len()); + match session.sync_actions(&active_action_sets) { + Err(err) => { + warn!("{}", err); + } + _ => {} + } +} + +pub trait Vec3Conv { + fn to_vec3(&self) -> Vec3; +} + +impl Vec3Conv for openxr::Vector3f { + fn to_vec3(&self) -> Vec3 { + Vec3::new(self.x, self.y, self.z) + } +} +pub trait QuatConv { + fn to_quat(&self) -> Quat; +} + +impl QuatConv for openxr::Quaternionf { + fn to_quat(&self) -> Quat { + Quat::from_xyzw(self.x, self.y, self.z, self.w) + } +} diff --git a/src/xr_input/oculus_touch.rs b/src/xr_input/oculus_touch.rs new file mode 100644 index 0000000..8c01817 --- /dev/null +++ b/src/xr_input/oculus_touch.rs @@ -0,0 +1,254 @@ +use crate::resources::{XrInstance, XrSession}; +use crate::xr_input::controllers::{Handed, Touchable}; +use crate::FutureXrResources; +use bevy::prelude::{Commands, Res, Resource}; +use openxr::{Action, ActionSet, AnyGraphics, Binding, Haptic, Instance, Posef, Session, Space}; +use std::any::Any; + +pub fn setup_oculus_controller( + mut commands: Commands, + instance: Res, + session: Res, +) { + let mut action_sets = vec![]; + let oculus_controller = OculusController::new( + Instance::clone(&instance), + Session::clone(&session), + &mut action_sets, + ) + .unwrap(); + session + .attach_action_sets(&action_sets.iter().map(|a| a).collect::>()) + .unwrap(); + commands.insert_resource(oculus_controller); + commands.insert_resource(ActionSets(action_sets)); +} + +#[derive(Resource, Clone)] +pub struct ActionSets(pub Vec); + +#[derive(Resource)] +pub struct OculusController { + pub grip_space: Handed, + pub aim_space: Handed, + pub grip_pose: Action, + pub aim_pose: Action, + pub squeeze: Action, + pub trigger: Touchable, + pub haptic_feedback: Action, + pub x_button: Touchable, + pub y_button: Touchable, + pub menu_button: Action, + pub a_button: Touchable, + pub b_button: Touchable, + pub thumbstick_x: Action, + pub thumbstick_y: Action, + pub thumbstick_touch: Action, + pub thumbstick_click: Action, + pub thumbrest_touch: Action, +} +impl OculusController { + pub fn new( + instance: Instance, + session: Session, + action_sets: &mut Vec, + ) -> anyhow::Result { + let action_set = + instance.create_action_set("oculus_input", "Oculus Touch Controller Input", 0)?; + let left_path = instance.string_to_path("/user/hand/left").unwrap(); + let right_path = instance.string_to_path("/user/hand/right").unwrap(); + let hands = [left_path, right_path]; + let grip_pose = action_set.create_action::("hand_pose", "Hand Pose", &hands)?; + let aim_pose = action_set.create_action::("pointer_pose", "Pointer Pose", &hands)?; + + let this = OculusController { + grip_space: Handed { + left: grip_pose.create_space(session.clone(), right_path, Posef::IDENTITY)?, + right: grip_pose.create_space(session.clone(), left_path, Posef::IDENTITY)?, + }, + aim_space: Handed { + left: aim_pose.create_space(session.clone(), right_path, Posef::IDENTITY)?, + right: aim_pose.create_space(session.clone(), left_path, Posef::IDENTITY)?, + }, + grip_pose, + aim_pose, + squeeze: action_set.create_action("squeeze", "Grip Pull", &hands)?, + trigger: Touchable { + inner: action_set.create_action("trigger", "Trigger Pull", &hands)?, + touch: action_set.create_action("trigger_touched", "Trigger Touch", &hands)?, + }, + haptic_feedback: action_set.create_action( + "haptic_feedback", + "Haptic Feedback", + &hands, + )?, + x_button: Touchable { + inner: action_set.create_action("x_button", "X Button", &[])?, + touch: action_set.create_action("x_button_touch", "X Button Touch", &[])?, + }, + y_button: Touchable { + inner: action_set.create_action("y_button", "Y Button", &[])?, + touch: action_set.create_action("y_button_touch", "Y Button Touch", &[])?, + }, + menu_button: action_set.create_action("menu_button", "Menu Button", &[])?, + a_button: Touchable { + inner: action_set.create_action("a_button", "A Button", &[])?, + touch: action_set.create_action("a_button_touch", "A Button Touch", &[])?, + }, + b_button: Touchable { + inner: action_set.create_action("b_button", "B Button", &[])?, + touch: action_set.create_action("b_button_touch", "B Button Touch", &[])?, + }, + thumbstick_x: action_set.create_action("thumbstick_x", "Thumbstick X", &hands)?, + thumbstick_y: action_set.create_action("thumbstick_y", "Thumbstick Y", &hands)?, + thumbstick_touch: action_set.create_action( + "thumbstick_touch", + "Thumbstick Touch", + &hands, + )?, + thumbstick_click: action_set.create_action( + "thumbstick_click", + "Thumbstick Click", + &hands, + )?, + thumbrest_touch: action_set.create_action( + "thumbrest_touch", + "Thumbrest Touch", + &hands, + )?, + }; + let i = instance; + i.suggest_interaction_profile_bindings( + i.string_to_path("/interaction_profiles/oculus/touch_controller")?, + &[ + Binding::new( + &this.grip_pose, + i.string_to_path("/user/hand/left/input/grip/pose")?, + ), + Binding::new( + &this.grip_pose, + i.string_to_path("/user/hand/right/input/grip/pose")?, + ), + Binding::new( + &this.aim_pose, + i.string_to_path("/user/hand/left/input/aim/pose")?, + ), + Binding::new( + &this.aim_pose, + i.string_to_path("/user/hand/left/input/aim/pose")?, + ), + Binding::new( + &this.squeeze, + i.string_to_path("/user/hand/left/input/squeeze/value")?, + ), + Binding::new( + &this.squeeze, + i.string_to_path("/user/hand/right/input/squeeze/value")?, + ), + Binding::new( + &this.trigger.inner, + i.string_to_path("/user/hand/right/input/trigger/value")?, + ), + Binding::new( + &this.trigger.inner, + i.string_to_path("/user/hand/left/input/trigger/value")?, + ), + Binding::new( + &this.trigger.touch, + i.string_to_path("/user/hand/right/input/trigger/touch")?, + ), + Binding::new( + &this.trigger.touch, + i.string_to_path("/user/hand/left/input/trigger/touch")?, + ), + Binding::new( + &this.haptic_feedback, + i.string_to_path("/user/hand/right/output/haptic")?, + ), + Binding::new( + &this.haptic_feedback, + i.string_to_path("/user/hand/left/output/haptic")?, + ), + Binding::new( + &this.x_button.inner, + i.string_to_path("/user/hand/left/input/x/click")?, + ), + Binding::new( + &this.x_button.touch, + i.string_to_path("/user/hand/left/input/x/touch")?, + ), + Binding::new( + &this.y_button.inner, + i.string_to_path("/user/hand/left/input/y/click")?, + ), + Binding::new( + &this.y_button.touch, + i.string_to_path("/user/hand/left/input/y/touch")?, + ), + Binding::new( + &this.menu_button, + i.string_to_path("/user/hand/left/input/menu/click")?, + ), + Binding::new( + &this.a_button.inner, + i.string_to_path("/user/hand/right/input/a/click")?, + ), + Binding::new( + &this.a_button.touch, + i.string_to_path("/user/hand/right/input/a/touch")?, + ), + Binding::new( + &this.b_button.inner, + i.string_to_path("/user/hand/right/input/b/click")?, + ), + Binding::new( + &this.b_button.touch, + i.string_to_path("/user/hand/right/input/b/touch")?, + ), + Binding::new( + &this.thumbstick_x, + i.string_to_path("/user/hand/left/input/thumbstick/x")?, + ), + Binding::new( + &this.thumbstick_x, + i.string_to_path("/user/hand/right/input/thumbstick/x")?, + ), + Binding::new( + &this.thumbstick_y, + i.string_to_path("/user/hand/left/input/thumbstick/y")?, + ), + Binding::new( + &this.thumbstick_y, + i.string_to_path("/user/hand/right/input/thumbstick/y")?, + ), + Binding::new( + &this.thumbstick_click, + i.string_to_path("/user/hand/left/input/thumbstick/click")?, + ), + Binding::new( + &this.thumbstick_click, + i.string_to_path("/user/hand/right/input/thumbstick/click")?, + ), + Binding::new( + &this.thumbstick_touch, + i.string_to_path("/user/hand/left/input/thumbstick/touch")?, + ), + Binding::new( + &this.thumbstick_touch, + i.string_to_path("/user/hand/right/input/thumbstick/touch")?, + ), + Binding::new( + &this.thumbrest_touch, + i.string_to_path("/user/hand/left/input/thumbrest/touch")?, + ), + Binding::new( + &this.thumbrest_touch, + i.string_to_path("/user/hand/right/input/thumbrest/touch")?, + ), + ], + )?; + + action_sets.push(action_set); + Ok(this) + } +} diff --git a/src/xr_input/xr_camera.rs b/src/xr_input/xr_camera.rs new file mode 100644 index 0000000..fbd2a90 --- /dev/null +++ b/src/xr_input/xr_camera.rs @@ -0,0 +1,239 @@ +use crate::xr_input::{QuatConv, Vec3Conv}; +use crate::{LEFT_XR_TEXTURE_HANDLE, RIGHT_XR_TEXTURE_HANDLE}; +use bevy::core_pipeline::tonemapping::{DebandDither, Tonemapping}; +use bevy::prelude::*; +use bevy::render::camera::{CameraProjection, CameraRenderGraph, RenderTarget}; +use bevy::render::primitives::Frustum; +use bevy::render::view::{ColorGrading, VisibleEntities}; +use openxr::Fovf; + +#[derive(Bundle)] +pub struct XrCamerasBundle { + pub left: XrCameraBundle, + pub right: XrCameraBundle, +} +impl XrCamerasBundle { + pub fn new() -> Self { + Self::default() + } +} +impl Default for XrCamerasBundle { + fn default() -> Self { + Self { + left: XrCameraBundle::new(Eye::Left), + right: XrCameraBundle::new(Eye::Right), + } + } +} + +#[derive(Bundle)] +pub struct XrCameraBundle { + pub camera: Camera, + pub camera_render_graph: CameraRenderGraph, + pub xr_projection: XRProjection, + pub visible_entities: VisibleEntities, + pub frustum: Frustum, + pub transform: Transform, + pub global_transform: GlobalTransform, + pub camera_3d: Camera3d, + pub tonemapping: Tonemapping, + pub dither: DebandDither, + pub color_grading: ColorGrading, + pub xr_camera_type: XrCameraType, +} +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Component)] +pub enum XrCameraType { + Xr(Eye), + Flatscreen, +} +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub enum Eye { + Left = 0, + Right = 1, +} + +impl XrCameraBundle { + pub fn new(eye: Eye) -> Self { + Self { + camera: Camera { + order: -1, + target: RenderTarget::TextureView(match eye { + Eye::Left => LEFT_XR_TEXTURE_HANDLE, + Eye::Right => RIGHT_XR_TEXTURE_HANDLE, + }), + viewport: None, + ..default() + }, + camera_render_graph: CameraRenderGraph::new(bevy::core_pipeline::core_3d::graph::NAME), + xr_projection: Default::default(), + visible_entities: Default::default(), + frustum: Default::default(), + transform: Default::default(), + global_transform: Default::default(), + camera_3d: Default::default(), + tonemapping: Default::default(), + dither: DebandDither::Enabled, + color_grading: Default::default(), + xr_camera_type: XrCameraType::Xr(eye), + } + } +} + +#[derive(Debug, Clone, Component, Reflect)] +#[reflect(Component, Default)] +pub struct XRProjection { + pub near: f32, + pub far: f32, + #[reflect(ignore)] + pub fov: Fovf, +} + +impl Default for XRProjection { + fn default() -> Self { + Self { + near: 0.1, + far: 1000., + fov: Default::default(), + } + } +} + +impl XRProjection { + pub fn new(near: f32, far: f32, fov: Fovf) -> Self { + XRProjection { near, far, fov } + } +} + +impl CameraProjection for XRProjection { + // ============================================================================= + // math code adapted from + // https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/master/src/common/xr_linear.h + // Copyright (c) 2017 The Khronos Group Inc. + // Copyright (c) 2016 Oculus VR, LLC. + // SPDX-License-Identifier: Apache-2.0 + // ============================================================================= + fn get_projection_matrix(&self) -> Mat4 { + // symmetric perspective for debugging + // let x_fov = (self.fov.angle_left.abs() + self.fov.angle_right.abs()); + // let y_fov = (self.fov.angle_up.abs() + self.fov.angle_down.abs()); + // return Mat4::perspective_infinite_reverse_rh(y_fov, x_fov / y_fov, self.near); + + let fov = self.fov; + let is_vulkan_api = false; // FIXME wgpu probably abstracts this + let near_z = self.near; + let far_z = -1.; // use infinite proj + // let far_z = self.far; + + let tan_angle_left = fov.angle_left.tan(); + let tan_angle_right = fov.angle_right.tan(); + + let tan_angle_down = fov.angle_down.tan(); + let tan_angle_up = fov.angle_up.tan(); + + let tan_angle_width = tan_angle_right - tan_angle_left; + + // Set to tanAngleDown - tanAngleUp for a clip space with positive Y + // down (Vulkan). Set to tanAngleUp - tanAngleDown for a clip space with + // positive Y up (OpenGL / D3D / Metal). + // const float tanAngleHeight = + // graphicsApi == GRAPHICS_VULKAN ? (tanAngleDown - tanAngleUp) : (tanAngleUp - tanAngleDown); + let tan_angle_height = if is_vulkan_api { + tan_angle_down - tan_angle_up + } else { + tan_angle_up - tan_angle_down + }; + + // Set to nearZ for a [-1,1] Z clip space (OpenGL / OpenGL ES). + // Set to zero for a [0,1] Z clip space (Vulkan / D3D / Metal). + // const float offsetZ = + // (graphicsApi == GRAPHICS_OPENGL || graphicsApi == GRAPHICS_OPENGL_ES) ? nearZ : 0; + // FIXME handle enum of graphics apis + let offset_z = 0.; + + let mut cols: [f32; 16] = [0.0; 16]; + + if far_z <= near_z { + // place the far plane at infinity + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -1.; + cols[14] = -(near_z + offset_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + + // bevy uses the _reverse_ infinite projection + // https://dev.theomader.com/depth-precision/ + let z_reversal = Mat4::from_cols_array_2d(&[ + [1f32, 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., -1., 0.], + [0., 0., 1., 1.], + ]); + + return z_reversal * Mat4::from_cols_array(&cols); + } else { + // normal projection + cols[0] = 2. / tan_angle_width; + cols[4] = 0.; + cols[8] = (tan_angle_right + tan_angle_left) / tan_angle_width; + cols[12] = 0.; + + cols[1] = 0.; + cols[5] = 2. / tan_angle_height; + cols[9] = (tan_angle_up + tan_angle_down) / tan_angle_height; + cols[13] = 0.; + + cols[2] = 0.; + cols[6] = 0.; + cols[10] = -(far_z + offset_z) / (far_z - near_z); + cols[14] = -(far_z * (near_z + offset_z)) / (far_z - near_z); + + cols[3] = 0.; + cols[7] = 0.; + cols[11] = -1.; + cols[15] = 0.; + } + + Mat4::from_cols_array(&cols) + } + + fn update(&mut self, _width: f32, _height: f32) {} + + fn far(&self) -> f32 { + self.far + } +} + +pub fn xr_camera_head_sync( + views: ResMut, + mut query: Query<(&mut Transform, &XrCameraType, &mut XRProjection)>, +) { + let mut f = || -> Option<()> { + for (mut transform, camera_type, mut xr_projection) in query.iter_mut() { + let view_idx = match camera_type { + XrCameraType::Xr(eye) => *eye as usize, + XrCameraType::Flatscreen => return None, + }; + let v = views.lock().unwrap(); + let view = v.get(view_idx)?; + xr_projection.fov = view.fov; + transform.rotation = view.pose.orientation.to_quat(); + transform.translation = view.pose.position.to_vec3(); + } + Some(()) + }; + let _ = f(); +}