Avii/bevy_oxr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add support for hand tracking and configurable reference spaces

Browse Source
This commit is contained in:
Schmarni
2024-05-07 03:34:10 +02:00
parent 3654b36e76
commit 063aef7fb5
17 changed files with 549 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
crates/bevy_openxr/src/openxr/features/handtracking.rs Normal file
View File

@@ -0,0 +1,176 @@
use bevy::prelude::*;
use bevy_xr::{
hands::{HandBone, HandBoneRadius},
session::{session_running, status_changed_to, XrStatus},
};
use openxr::SpaceLocationFlags;
use crate::{
init::OxrTrackingRoot, reference_space::{OxrPrimaryReferenceSpace, OxrReferenceSpace}, resources::{OxrSession, OxrTime}
};
pub struct HandTrackingPlugin {
default_hands: bool,
}
impl Default for HandTrackingPlugin {
fn default() -> Self {
Self {
default_hands: true,
}
}
}
impl Plugin for HandTrackingPlugin {
fn build(&self, app: &mut App) {
app.add_systems(PreUpdate, locate_hands.run_if(session_running));
if self.default_hands {
app.add_systems(
PreUpdate,
clean_up_default_hands.run_if(status_changed_to(XrStatus::Exiting)),
);
app.add_systems(
PostUpdate,
spawn_default_hands.run_if(status_changed_to(XrStatus::Ready)),
);
}
}
}
fn spawn_default_hands(
mut cmds: Commands,
session: Res<OxrSession>,
root: Query<Entity, With<OxrTrackingRoot>>,
) {
info!("spawning hands");
let Ok(root) = root.get_single() else {
error!("unable to get tracking root, skipping hand creation");
return;
};
let tracker_left = match session.create_hand_tracker(openxr::HandEXT::LEFT) {
Ok(t) => t,
Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
warn!("Handtracking Extension not loaded, Unable to create Handtracker!");
return;
}
Err(err) => {
warn!("Error while creating Handtracker: {}", err.to_string());
return;
}
};
let tracker_right = match session.create_hand_tracker(openxr::HandEXT::RIGHT) {
Ok(t) => t,
Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
warn!("Handtracking Extension not loaded, Unable to create Handtracker!");
return;
}
Err(err) => {
warn!("Error while creating Handtracker: {}", err.to_string());
return;
}
};
let mut left_bones = [Entity::PLACEHOLDER; 26];
let mut right_bones = [Entity::PLACEHOLDER; 26];
for bone in HandBone::get_all_bones() {
let bone_left = cmds
.spawn((SpatialBundle::default(), bone, HandBoneRadius(0.0)))
.id();
let bone_right = cmds
.spawn((SpatialBundle::default(), bone, HandBoneRadius(0.0)))
.id();
cmds.entity(root).push_children(&[bone_right]);
left_bones[bone as usize] = bone_left;
right_bones[bone as usize] = bone_right;
}
cmds.spawn((
DefaultHandTracker,
OxrHandTracker(tracker_left),
OxrHandBoneEntities(left_bones),
));
cmds.spawn((
DefaultHandTracker,
OxrHandTracker(tracker_right),
OxrHandBoneEntities(right_bones),
));
}
#[derive(Component)]
struct DefaultHandTracker;
#[derive(Component)]
struct DefaultHandBones;
#[allow(clippy::type_complexity)]
fn clean_up_default_hands(
mut cmds: Commands,
query: Query<Entity, Or<(With<DefaultHandTracker>, With<DefaultHandBones>)>>,
) {
for e in &query {
cmds.entity(e).despawn();
}
}
#[derive(Deref, DerefMut, Component, Clone, Copy)]
pub struct OxrHandBoneEntities(pub [Entity; 26]);
#[derive(Deref, DerefMut, Component)]
pub struct OxrHandTracker(pub openxr::HandTracker);
fn locate_hands(
default_ref_space: Res<OxrPrimaryReferenceSpace>,
time: Res<OxrTime>,
tracker_query: Query<(
&OxrHandTracker,
Option<&OxrReferenceSpace>,
&OxrHandBoneEntities,
)>,
mut bone_query: Query<(&HandBone, &mut HandBoneRadius, &mut Transform)>,
) {
info!("updating hands 0 ");
for (tracker, ref_space, hand_entities) in &tracker_query {
info!("updating hands 1 ");
let ref_space = ref_space.map(|v| &v.0).unwrap_or(&default_ref_space.0);
// relate_hand_joints also provides velocities
let joints = match ref_space.locate_hand_joints(tracker, **time) {
Ok(Some(v)) => v,
Ok(None) => continue,
Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
error!("HandTracking Extension not loaded");
continue;
}
Err(err) => {
warn!("Error while locating hand joints: {}", err.to_string());
continue;
}
};
let bone_entities = match bone_query.get_many_mut(hand_entities.0) {
Ok(v) => v,
Err(err) => {
warn!("unable to get entities, {}", err);
continue;
}
};
for (bone, mut bone_radius, mut transform) in bone_entities {
info!("updating hands");
let joint = joints[*bone as usize];
**bone_radius = joint.radius;
if joint
.location_flags
.contains(SpaceLocationFlags::POSITION_VALID)
{
transform.translation.x = joint.pose.position.x;
transform.translation.y = joint.pose.position.y;
transform.translation.z = joint.pose.position.z;
}
if joint
.location_flags
.contains(SpaceLocationFlags::ORIENTATION_VALID)
{
transform.rotation.x = joint.pose.orientation.x;
transform.rotation.y = joint.pose.orientation.y;
transform.rotation.z = joint.pose.orientation.z;
transform.rotation.w = joint.pose.orientation.w;
}
}
}
}

1
crates/bevy_openxr/src/openxr/features/mod.rs
View File

@@ -1,2 +1,3 @@
#[cfg(feature = "passthrough")]
pub mod passthrough;
pub mod handtracking;

90
crates/bevy_openxr/src/openxr/helper_traits.rs Normal file
View File

@@ -0,0 +1,90 @@
use bevy::prelude::*;
pub trait ToPosef {
fn to_posef(&self) -> openxr::Posef;
}
pub trait ToTransform {
fn to_transform(&self) -> Transform;
}
pub trait ToQuaternionf {
fn to_quaternionf(&self) -> openxr::Quaternionf;
}
pub trait ToQuat {
fn to_quat(&self) -> Quat;
}
pub trait ToVector3f {
fn to_vector3f(&self) -> openxr::Vector3f;
}
pub trait ToVec3 {
fn to_vec3(&self) -> Vec3;
}
pub trait ToVector2f {
fn to_vector2f(&self) -> openxr::Vector2f;
}
pub trait ToVec2 {
fn to_vec2(&self) -> Vec2;
}
impl ToPosef for Transform {
fn to_posef(&self) -> openxr::Posef {
openxr::Posef {
orientation: self.rotation.to_quaternionf(),
position: self.translation.to_vector3f(),
}
}
}
impl ToTransform for openxr::Posef {
fn to_transform(&self) -> Transform {
Transform::from_translation(self.position.to_vec3())
.with_rotation(self.orientation.to_quat())
}
}
impl ToQuaternionf for Quat {
fn to_quaternionf(&self) -> openxr::Quaternionf {
openxr::Quaternionf {
x: self.x,
y: self.y,
z: self.z,
w: self.w,
}
}
}
impl ToQuat for openxr::Quaternionf {
fn to_quat(&self) -> Quat {
Quat::from_xyzw(self.x, self.y, self.z, self.w)
}
}
impl ToVector3f for Vec3 {
fn to_vector3f(&self) -> openxr::Vector3f {
openxr::Vector3f {
x: self.x,
y: self.y,
z: self.z,
}
}
}
impl ToVec3 for openxr::Vector3f {
fn to_vec3(&self) -> Vec3 {
Vec3 {
x: self.x,
y: self.y,
z: self.z,
}
}
}
impl ToVector2f for Vec2 {
fn to_vector2f(&self) -> openxr::Vector2f {
openxr::Vector2f {
x: self.x,
y: self.y,
}
}
}
impl ToVec2 for openxr::Vector2f {
fn to_vec2(&self) -> Vec2 {
Vec2 {
x: self.x,
y: self.y,
}
}
}

27
crates/bevy_openxr/src/openxr/init.rs
View File

@@ -28,6 +28,7 @@ use bevy_xr::session::XrStatusChanged;
use crate::error::OxrError;
use crate::graphics::*;
use crate::reference_space::OxrPrimaryReferenceSpace;
use crate::resources::*;
use crate::types::*;
@@ -39,6 +40,8 @@ pub fn session_started(started: Option<Res<OxrSessionStarted>>) -> bool {
pub enum OxrPreUpdateSet {
PollEvents,
HandleEvents,
UpdateCriticalComponents,
UpdateNonCriticalComponents,
}
pub struct OxrInitPlugin {
@@ -90,7 +93,7 @@ impl Plugin for OxrInitPlugin {
))
.add_systems(First, reset_per_frame_resources)
.add_systems(
PreUpdate,
First,
(
poll_events
.run_if(session_available)
@@ -167,7 +170,9 @@ impl Plugin for OxrInitPlugin {
(
OxrPreUpdateSet::PollEvents.before(handle_session),
OxrPreUpdateSet::HandleEvents.after(handle_session),
),
OxrPreUpdateSet::UpdateCriticalComponents,
OxrPreUpdateSet::UpdateNonCriticalComponents,
).chain(),
);
let session_started = OxrSessionStarted::default();
@@ -293,7 +298,6 @@ fn init_xr_session(
OxrSwapchain,
OxrSwapchainImages,
OxrGraphicsInfo,
OxrStage,
)> {
let (session, frame_waiter, frame_stream) =
unsafe { instance.create_session(system_id, graphics_info)? };
@@ -395,11 +399,6 @@ fn init_xr_session(
}
.ok_or(OxrError::NoAvailableBackend)?;
let stage = OxrStage(
session
.create_reference_space(openxr::ReferenceSpaceType::STAGE, openxr::Posef::IDENTITY)?
.into(),
);
let graphics_info = OxrGraphicsInfo {
blend_mode,
@@ -414,7 +413,6 @@ fn init_xr_session(
swapchain,
images,
graphics_info,
stage,
))
}
@@ -426,7 +424,6 @@ struct OxrRenderResources {
swapchain: OxrSwapchain,
images: OxrSwapchainImages,
graphics_info: OxrGraphicsInfo,
stage: OxrStage,
}
pub fn create_xr_session(
@@ -442,19 +439,17 @@ pub fn create_xr_session(
**system_id,
create_info.clone(),
) {
Ok((session, frame_waiter, frame_stream, swapchain, images, graphics_info, stage)) => {
Ok((session, frame_waiter, frame_stream, swapchain, images, graphics_info)) => {
commands.insert_resource(session.clone());
commands.insert_resource(frame_waiter);
commands.insert_resource(images.clone());
commands.insert_resource(graphics_info.clone());
commands.insert_resource(stage.clone());
commands.insert_resource(OxrRenderResources {
session,
frame_stream,
swapchain,
images,
graphics_info,
stage,
});
}
Err(e) => error!("Failed to initialize XrSession: {e}"),
@@ -483,7 +478,6 @@ pub fn transfer_xr_resources(mut commands: Commands, mut world: ResMut<MainWorld
swapchain,
images,
graphics_info,
stage,
}) = world.remove_resource()
else {
return;
@@ -494,7 +488,6 @@ pub fn transfer_xr_resources(mut commands: Commands, mut world: ResMut<MainWorld
commands.insert_resource(swapchain);
commands.insert_resource(images);
commands.insert_resource(graphics_info);
commands.insert_resource(stage);
}
/// Polls any OpenXR events and handles them accordingly
@@ -547,14 +540,14 @@ pub fn destroy_xr_session(mut commands: Commands) {
commands.remove_resource::<OxrFrameWaiter>();
commands.remove_resource::<OxrSwapchainImages>();
commands.remove_resource::<OxrGraphicsInfo>();
commands.remove_resource::<OxrStage>();
commands.remove_resource::<OxrPrimaryReferenceSpace>();
commands.insert_resource(OxrCleanupSession(true));
}
pub fn destroy_xr_session_render(world: &mut World) {
world.remove_resource::<OxrSwapchain>();
world.remove_resource::<OxrFrameStream>();
world.remove_resource::<OxrStage>();
world.remove_resource::<OxrPrimaryReferenceSpace>();
world.remove_resource::<OxrSwapchainImages>();
world.remove_resource::<OxrGraphicsInfo>();
world.remove_resource::<OxrSession>();

3
crates/bevy_openxr/src/openxr/layer_builder.rs
View File

@@ -4,6 +4,7 @@ use bevy::ecs::world::World;
use openxr::{sys, CompositionLayerFlags, Fovf, Posef, Rect2Di, Space};
use crate::graphics::graphics_match;
use crate::reference_space::OxrPrimaryReferenceSpace;
use crate::resources::*;
pub trait LayerProvider {
@@ -16,7 +17,7 @@ pub struct PassthroughLayer;
impl LayerProvider for ProjectionLayer {
fn get<'a>(&self, world: &'a World) -> Box<dyn CompositionLayer<'a> + 'a> {
let stage = world.resource::<OxrStage>();
let stage = world.resource::<OxrPrimaryReferenceSpace>();
let openxr_views = world.resource::<OxrViews>();
let swapchain = world.resource::<OxrSwapchain>();
let graphics_info = world.resource::<OxrGraphicsInfo>();

9
crates/bevy_openxr/src/openxr/mod.rs
View File

@@ -23,6 +23,8 @@ pub mod resources;
pub mod types;
pub mod action_binding;
pub mod action_set_attaching;
pub mod reference_space;
pub mod helper_traits;
pub fn add_xr_plugins<G: PluginGroup>(plugins: G) -> PluginGroupBuilder {
plugins
@@ -50,17 +52,14 @@ pub fn add_xr_plugins<G: PluginGroup>(plugins: G) -> PluginGroupBuilder {
.add(action_binding::OxrActionBindingPlugin)
// .add(XrActionPlugin)
.set(WindowPlugin {
#[cfg(not(target_os = "android"))]
primary_window: Some(Window {
transparent: true,
present_mode: PresentMode::AutoNoVsync,
// title: self.app_info.name.clone(),
..default()
}),
#[cfg(target_os = "android")]
primary_window: None, // ?
#[cfg(target_os = "android")]
exit_condition: bevy::window::ExitCondition::DontExit,
// #[cfg(target_os = "android")]
// exit_condition: bevy::window::ExitCondition::DontExit,
#[cfg(target_os = "android")]
close_when_requested: true,
..default()

60
crates/bevy_openxr/src/openxr/reference_space.rs Normal file
View File

@@ -0,0 +1,60 @@
use std::sync::Arc;
use bevy::{
prelude::*,
render::extract_resource::{ExtractResource, ExtractResourcePlugin},
};
use bevy_xr::session::{status_changed_to, XrStatus};
use crate::{init::OxrPreUpdateSet, resources::OxrSession};
pub struct OxrReferenceSpacePlugin {
default_primary_ref_space: openxr::ReferenceSpaceType,
}
impl Default for OxrReferenceSpacePlugin {
fn default() -> Self {
Self {
default_primary_ref_space: openxr::ReferenceSpaceType::LOCAL_FLOOR_EXT,
}
}
}
#[derive(Resource)]
struct OxrPrimaryReferenceSpaceType(openxr::ReferenceSpaceType);
// TODO: this will keep the session alive so we need to remove this in the render world too
/// The Default Reference space used for locating things
#[derive(Resource, Deref, ExtractResource, Clone)]
pub struct OxrPrimaryReferenceSpace(pub Arc<openxr::Space>);
/// The Reference space used for locating spaces on this entity
#[derive(Component)]
pub struct OxrReferenceSpace(pub openxr::Space);
impl Plugin for OxrReferenceSpacePlugin {
fn build(&self, app: &mut App) {
app.insert_resource(OxrPrimaryReferenceSpaceType(self.default_primary_ref_space));
app.add_plugins(ExtractResourcePlugin::<OxrPrimaryReferenceSpace>::default());
app.add_systems(
PreUpdate,
set_primary_ref_space
.run_if(status_changed_to(XrStatus::Ready))
.in_set(OxrPreUpdateSet::UpdateCriticalComponents),
);
}
}
fn set_primary_ref_space(
session: Res<OxrSession>,
space_type: Res<OxrPrimaryReferenceSpaceType>,
mut cmds: Commands,
) {
match session.create_reference_space(space_type.0, openxr::Posef::IDENTITY) {
Ok(space) => {
cmds.insert_resource(OxrPrimaryReferenceSpace(Arc::new(space)));
}
Err(openxr::sys::Result::ERROR_EXTENSION_NOT_PRESENT) => {
error!("Required Extension for Reference Space not loaded");
}
Err(err) => error!("Error while creating reference space: {}", err.to_string()),
};
}

6
crates/bevy_openxr/src/openxr/render.rs
View File

@@ -12,7 +12,7 @@ use bevy::{
use bevy_xr::camera::{XrCamera, XrCameraBundle, XrProjection};
use openxr::ViewStateFlags;
use crate::resources::*;
use crate::{reference_space::OxrPrimaryReferenceSpace, resources::*};
use crate::{
init::{session_started, OxrPreUpdateSet},
layer_builder::ProjectionLayer,
@@ -112,7 +112,7 @@ pub fn wait_frame(mut frame_waiter: ResMut<OxrFrameWaiter>, mut commands: Comman
pub fn locate_views(
session: Res<OxrSession>,
stage: Res<OxrStage>,
ref_space: Res<OxrPrimaryReferenceSpace>,
time: Res<OxrTime>,
mut openxr_views: ResMut<OxrViews>,
) {
@@ -121,7 +121,7 @@ pub fn locate_views(
.locate_views(
openxr::ViewConfigurationType::PRIMARY_STEREO,
**time,
&stage,
&ref_space,
)
.expect("Failed to locate views");
if openxr_views.len() != xr_views.len() {

4
crates/bevy_openxr/src/openxr/resources.rs
View File

@@ -377,8 +377,8 @@ impl OxrSwapchain {
pub struct OxrSwapchainImages(pub Arc<Vec<wgpu::Texture>>);
/// Thread safe wrapper around [openxr::Space] representing the stage.
#[derive(Deref, Clone, Resource)]
pub struct OxrStage(pub Arc<openxr::Space>);
// #[derive(Deref, Clone, Resource)]
// pub struct OxrStage(pub Arc<openxr::Space>);
/// Stores the latest generated [OxrViews]
#[derive(Clone, Resource, ExtractResource, Deref, DerefMut)]