Avii/bevy_oxr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #41 from Schmarni-Dev/hand_refactor

Browse Source 
Hand refactor
This commit is contained in:
Malek
2023-11-20 10:28:22 -08:00
committed by GitHub
parent 2f91b5cbb1 f47375b97d
commit f76e81c0e0
20 changed files with 1942 additions and 1393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@@ -1,3 +1,4 @@
/target
/Cargo.lock
**/target
**/Cargo.lock
**/runtime_libs/arm64-v8a/*
\.DS_Store

2
examples/demo/Cargo.toml
View File

@@ -4,7 +4,7 @@ version = "0.1.0"
edition = "2021"
[lib]
crate-type = ["lib", "cdylib"]
crate-type = ["rlib", "cdylib"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

24
examples/demo/Cargo_back.toml Normal file
View File

@@ -0,0 +1,24 @@
[package]
name = "demo"
version = "0.1.0"
edition = "2021"
[lib]
crate-type = ["cdylib", "lib"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bevy = { git = "https://github.com/bevyengine/bevy.git" }
# bevy = "0.11.3"
# default-features is false because it for some reason complains when trying to statically link openxr
bevy_oxr = { path = "../../" }
# bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"}
bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", version = "0.22.0", branch = "bevy-0.12" }
# bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
color-eyre = "0.6.2"
[profile.release]
lto = "fat"
codegen-units = 1
panic = "abort"

11
examples/demo/Readme.md Normal file
View File

@@ -0,0 +1,11 @@
## setup
install xbuild ```cargo install --git https://github.com/rust-mobile/xbuild```
run ```x doctor``` and install all required depencencies
download the [openxr loader](https://developer.oculus.com/downloads/package/oculus-openxr-mobile-sdk/) and put it in the runtime_libs/arm64-v8a folder
## how to run
run ```x devices```
and get the device name that looks something like this ```adb:1WD523S``` (probably a bit longer)
then ```run x run --release```

6
examples/demo/src/lib.rs
View File

@@ -18,9 +18,9 @@ use bevy_oxr::{
input::XrInput,
resources::{XrFrameState, XrInstance, XrSession},
xr_input::{
actions::XrActionSets,
debug_gizmos::OpenXrDebugRenderer,
hand::{HandBone, HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
hands::common::{ HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
hands::HandBone,
interactions::{
draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
update_interactable_states, InteractionEvent, Touched, XRDirectInteractor,
@@ -29,7 +29,7 @@ use bevy_oxr::{
oculus_touch::OculusController,
prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig},
trackers::{OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker},
Hand,
Hand, actions::XrActionSets,
},
DefaultXrPlugins,
};

609
examples/demo/src/lib_bak.rs Normal file
View File

@@ -0,0 +1,609 @@
use bevy::{
diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin},
log::info,
prelude::{
*,
bevy_main, default, shape, App, Assets, Color, Commands, Component, Event, EventReader,
EventWriter, GlobalTransform, IntoSystemConfigs, IntoSystemSetConfigs, Mesh, PbrBundle,
PostUpdate, Query, Res, ResMut, Resource, SpatialBundle, StandardMaterial, Startup,
Transform, Update, With, Without,
},
time::{Time, Timer},
transform::TransformSystem,
};
use bevy_openxr::{
input::XrInput,
resources::{XrFrameState, XrInstance, XrSession},
xr_input::{
debug_gizmos::OpenXrDebugRenderer,
hand::{HandBone, HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
interactions::{
draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
update_interactable_states, InteractionEvent, Touched, XRDirectInteractor,
XRInteractable, XRInteractableState, XRInteractorState, XRSelection,
},
oculus_touch::OculusController,
prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig},
trackers::{OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker},
Hand,
},
DefaultXrPlugins,
};
mod setup;
use crate::setup::setup_scene;
use bevy_rapier3d::prelude::*;
#[bevy_main]
pub fn main() {
color_eyre::install().unwrap();
info!("Running bevy_openxr demo");
let mut app = App::new();
app
//lets get the usual diagnostic stuff added
.add_plugins(LogDiagnosticsPlugin::default())
.add_plugins(FrameTimeDiagnosticsPlugin)
//lets get the xr defaults added
.add_plugins(DefaultXrPlugins)
//lets add the debug renderer for the controllers
.add_plugins(OpenXrDebugRenderer)
//rapier goes here
.add_plugins(RapierPhysicsPlugin::<NoUserData>::default().with_default_system_setup(false))
.add_plugins(RapierDebugRenderPlugin::default())
//lets setup the starting scene
.add_systems(Startup, setup_scene)
.add_systems(Startup, spawn_controllers_example) //you need to spawn controllers or it crashes TODO:: Fix this
//add locomotion
.add_systems(Update, proto_locomotion)
.insert_resource(PrototypeLocomotionConfig::default())
//lets add the interaction systems
.add_event::<InteractionEvent>()
.add_systems(Update, prototype_interaction_input)
.add_systems(Update, interactions.before(update_interactable_states))
.add_systems(Update, update_interactable_states)
.add_systems(
Update,
socket_interactions.before(update_interactable_states),
)
//add the grabbable system
.add_systems(Update, update_grabbables.after(update_interactable_states))
//draw the interaction gizmos
.add_systems(
Update,
draw_interaction_gizmos.after(update_interactable_states),
)
.add_systems(Update, draw_socket_gizmos.after(update_interactable_states))
//add our cube spawning system
.add_event::<SpawnCubeRequest>()
.insert_resource(SpawnCubeTimer(Timer::from_seconds(
0.25,
bevy::time::TimerMode::Once,
)))
.add_systems(Update, request_cube_spawn)
.add_systems(Update, cube_spawner.after(request_cube_spawn))
//test capsule
.add_systems(Startup, spawn_capsule)
//physics hands
.add_plugins(OpenXrHandInput)
.add_plugins(HandInputDebugRenderer)
.add_systems(Startup, spawn_physics_hands)
.add_systems(Update, update_physics_hands);
//configure rapier sets
app.configure_sets(
PostUpdate,
(
PhysicsSet::SyncBackend,
PhysicsSet::StepSimulation,
PhysicsSet::Writeback,
)
.chain()
.before(TransformSystem::TransformPropagate),
);
//add rapier systems
app.add_systems(
PostUpdate,
(
RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::SyncBackend)
.in_set(PhysicsSet::SyncBackend),
(
RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::StepSimulation),
// despawn_one_box,
)
.in_set(PhysicsSet::StepSimulation),
RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::Writeback)
.in_set(PhysicsSet::Writeback),
),
);
app.run();
}
fn spawn_controllers_example(mut commands: Commands) {
//left hand
commands.spawn((
OpenXRLeftController,
OpenXRController,
OpenXRTracker,
SpatialBundle::default(),
XRDirectInteractor,
XRInteractorState::default(),
XRSelection::default(),
));
//right hand
commands.spawn((
OpenXRRightController,
OpenXRController,
OpenXRTracker,
SpatialBundle::default(),
XRDirectInteractor,
XRInteractorState::default(),
XRSelection::default(),
));
}
fn spawn_capsule(
mut commands: Commands,
mut meshes: ResMut<Assets<Mesh>>,
mut materials: ResMut<Assets<StandardMaterial>>,
) {
commands.spawn((
PbrBundle {
mesh: meshes.add(Mesh::from(shape::Capsule {
radius: 0.033,
depth: 0.115,
..default()
})),
material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
transform: Transform::from_xyz(0.0, 2.0, 0.0),
..default()
},
// Collider::capsule_y(0.0575, 0.034),
Collider::capsule(
Vec3 {
x: 0.0,
y: -0.0575,
z: 0.0,
},
Vec3 {
x: 0.0,
y: 0.0575,
z: 0.0,
},
0.034,
),
RigidBody::Dynamic,
));
}
#[derive(Component, PartialEq, Debug, Clone, Copy)]
pub enum PhysicsHandBone {
Palm,
Wrist,
ThumbMetacarpal,
ThumbProximal,
ThumbDistal,
ThumbTip,
IndexMetacarpal,
IndexProximal,
IndexIntermediate,
IndexDistal,
IndexTip,
MiddleMetacarpal,
MiddleProximal,
MiddleIntermediate,
MiddleDistal,
MiddleTip,
RingMetacarpal,
RingProximal,
RingIntermediate,
RingDistal,
RingTip,
LittleMetacarpal,
LittleProximal,
LittleIntermediate,
LittleDistal,
LittleTip,
}
#[derive(Component, PartialEq)]
pub enum BoneInitState {
True,
False,
}
fn spawn_physics_hands(mut commands: Commands) {
//here we go
let hands = [Hand::Left, Hand::Right];
let bones = [
PhysicsHandBone::Palm,
PhysicsHandBone::Wrist,
PhysicsHandBone::ThumbMetacarpal,
PhysicsHandBone::ThumbProximal,
PhysicsHandBone::ThumbDistal,
PhysicsHandBone::ThumbTip,
PhysicsHandBone::IndexMetacarpal,
PhysicsHandBone::IndexProximal,
PhysicsHandBone::IndexIntermediate,
PhysicsHandBone::IndexDistal,
PhysicsHandBone::IndexTip,
PhysicsHandBone::MiddleMetacarpal,
PhysicsHandBone::MiddleProximal,
PhysicsHandBone::MiddleIntermediate,
PhysicsHandBone::MiddleDistal,
PhysicsHandBone::MiddleTip,
PhysicsHandBone::RingMetacarpal,
PhysicsHandBone::RingProximal,
PhysicsHandBone::RingIntermediate,
PhysicsHandBone::RingDistal,
PhysicsHandBone::RingTip,
PhysicsHandBone::LittleMetacarpal,
PhysicsHandBone::LittleProximal,
PhysicsHandBone::LittleIntermediate,
PhysicsHandBone::LittleDistal,
PhysicsHandBone::LittleTip,
];
//lets just do the Right ThumbMetacarpal for now
//i dont understand the groups yet
let self_group = Group::GROUP_1;
let interaction_group = Group::ALL;
let radius = 0.010;
for hand in hands.iter() {
for bone in bones.iter() {
//spawn the thing
commands.spawn((
SpatialBundle::default(),
Collider::capsule(
Vec3 {
x: 0.0,
y: -0.0575,
z: 0.0,
},
Vec3 {
x: 0.0,
y: 0.0575,
z: 0.0,
},
radius,
),
RigidBody::KinematicPositionBased,
// CollisionGroups::new(self_group, interaction_group),
// SolverGroups::new(self_group, interaction_group),
bone.clone(),
BoneInitState::False,
hand.clone(),
));
}
}
}
fn update_physics_hands(
hands_res: Option<Res<HandsResource>>,
mut bone_query: Query<(
&mut Transform,
&mut Collider,
&PhysicsHandBone,
&mut BoneInitState,
&Hand,
)>,
hand_query: Query<(&Transform, &HandBone, &Hand, Without<PhysicsHandBone>)>,
) {
//sanity check do we even have hands?
match hands_res {
Some(res) => {
//config stuff
let radius = 0.010;
for mut bone in bone_query.iter_mut() {
let hand_res = match bone.4 {
Hand::Left => res.left,
Hand::Right => res.right,
};
//lets just do the Right ThumbMetacarpal for now
let result = get_start_and_end_entities(hand_res, bone.2);
if let Some((start_entity, end_entity)) = result {
//now we need their transforms
let start_components = hand_query.get(start_entity);
let end_components = hand_query.get(end_entity);
let direction = end_components.unwrap().0.translation
- start_components.unwrap().0.translation;
if direction.length() < 0.001 {
//i hate this but we need to skip init if the length is zero
return;
}
match *bone.3 {
BoneInitState::True => {
//if we are init then we just move em?
*bone.0 = start_components
.unwrap()
.0
.clone()
.looking_at(end_components.unwrap().0.translation, Vec3::Y);
}
BoneInitState::False => {
//build a new collider?
*bone.1 = Collider::capsule(
Vec3::splat(0.0),
Vec3 {
x: 0.0,
y: 0.0,
z: -direction.length(),
},
radius,
);
*bone.3 = BoneInitState::True;
}
}
}
}
}
None => info!("hand states resource not initialized yet"),
}
}
fn get_start_and_end_entities(
hand_res: HandResource,
bone: &PhysicsHandBone,
) -> Option<(Entity, Entity)> {
match bone {
PhysicsHandBone::Palm => return None,
PhysicsHandBone::Wrist => return None,
PhysicsHandBone::ThumbMetacarpal => {
return Some((hand_res.thumb.metacarpal, hand_res.thumb.proximal))
}
PhysicsHandBone::ThumbProximal => {
return Some((hand_res.thumb.proximal, hand_res.thumb.distal))
}
PhysicsHandBone::ThumbDistal => return Some((hand_res.thumb.distal, hand_res.thumb.tip)),
PhysicsHandBone::ThumbTip => return None,
PhysicsHandBone::IndexMetacarpal => {
return Some((hand_res.index.metacarpal, hand_res.index.proximal))
}
PhysicsHandBone::IndexProximal => {
return Some((hand_res.index.proximal, hand_res.index.intermediate))
}
PhysicsHandBone::IndexIntermediate => {
return Some((hand_res.index.intermediate, hand_res.index.distal))
}
PhysicsHandBone::IndexDistal => return Some((hand_res.index.distal, hand_res.index.tip)),
PhysicsHandBone::IndexTip => return None,
PhysicsHandBone::MiddleMetacarpal => {
return Some((hand_res.middle.metacarpal, hand_res.middle.proximal))
}
PhysicsHandBone::MiddleProximal => {
return Some((hand_res.middle.proximal, hand_res.middle.intermediate))
}
PhysicsHandBone::MiddleIntermediate => {
return Some((hand_res.middle.intermediate, hand_res.middle.distal))
}
PhysicsHandBone::MiddleDistal => {
return Some((hand_res.middle.distal, hand_res.middle.tip))
}
PhysicsHandBone::MiddleTip => return None,
PhysicsHandBone::RingMetacarpal => {
return Some((hand_res.ring.metacarpal, hand_res.ring.proximal))
}
PhysicsHandBone::RingProximal => {
return Some((hand_res.ring.proximal, hand_res.ring.intermediate))
}
PhysicsHandBone::RingIntermediate => {
return Some((hand_res.ring.intermediate, hand_res.ring.distal))
}
PhysicsHandBone::RingDistal => return Some((hand_res.ring.distal, hand_res.ring.tip)),
PhysicsHandBone::RingTip => return None,
PhysicsHandBone::LittleMetacarpal => {
return Some((hand_res.little.metacarpal, hand_res.little.proximal))
}
PhysicsHandBone::LittleProximal => {
return Some((hand_res.little.proximal, hand_res.little.intermediate))
}
PhysicsHandBone::LittleIntermediate => {
return Some((hand_res.little.intermediate, hand_res.little.distal))
}
PhysicsHandBone::LittleDistal => {
return Some((hand_res.little.distal, hand_res.little.tip))
}
PhysicsHandBone::LittleTip => return None,
};
}
fn get_hand_res(res: &Res<'_, HandsResource>, hand: Hand) -> HandResource {
match hand {
Hand::Left => res.left.clone(),
Hand::Right => res.right.clone(),
}
}
#[derive(Event, Default)]
pub struct SpawnCubeRequest;
#[derive(Resource)]
pub struct SpawnCubeTimer(Timer);
fn request_cube_spawn(
oculus_controller: Res<OculusController>,
frame_state: Res<XrFrameState>,
xr_input: Res<XrInput>,
instance: Res<XrInstance>,
session: Res<XrSession>,
mut writer: EventWriter<SpawnCubeRequest>,
time: Res<Time>,
mut timer: ResMut<SpawnCubeTimer>,
) {
timer.0.tick(time.delta());
if timer.0.finished() {
//lock frame
let frame_state = *frame_state.lock().unwrap();
//get controller
let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
//get controller triggers
let left_main_button = controller.a_button();
if left_main_button {
writer.send(SpawnCubeRequest::default());
timer.0.reset();
}
let right_main_button = controller.x_button();
if right_main_button {
writer.send(SpawnCubeRequest::default());
timer.0.reset();
}
}
}
fn cube_spawner(
mut commands: Commands,
mut meshes: ResMut<Assets<Mesh>>,
mut materials: ResMut<Assets<StandardMaterial>>,
mut events: EventReader<SpawnCubeRequest>,
) {
for request in events.iter() {
// cube
commands.spawn((
PbrBundle {
mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
transform: Transform::from_xyz(0.0, 1.0, 0.0),
..default()
},
RigidBody::Dynamic,
Collider::cuboid(0.05, 0.05, 0.05),
ColliderDebugColor(Color::hsl(220.0, 1.0, 0.3)),
XRInteractable,
XRInteractableState::default(),
Grabbable,
Touched(false),
));
}
}
//TODO: find a real place for this
fn prototype_interaction_input(
oculus_controller: Res<OculusController>,
frame_state: Res<XrFrameState>,
xr_input: Res<XrInput>,
instance: Res<XrInstance>,
session: Res<XrSession>,
mut right_interactor_query: Query<
(&mut XRInteractorState),
(
With<XRDirectInteractor>,
With<OpenXRRightController>,
Without<OpenXRLeftController>,
),
>,
mut left_interactor_query: Query<
(&mut XRInteractorState),
(
With<XRDirectInteractor>,
With<OpenXRLeftController>,
Without<OpenXRRightController>,
),
>,
) {
//lock frame
let frame_state = *frame_state.lock().unwrap();
//get controller
let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
//get controller triggers
let left_trigger = controller.trigger(Hand::Left);
let right_trigger = controller.trigger(Hand::Right);
//get the interactors and do state stuff
let mut left_state = left_interactor_query.single_mut();
if left_trigger > 0.8 {
*left_state = XRInteractorState::Selecting;
} else {
*left_state = XRInteractorState::Idle;
}
let mut right_state = right_interactor_query.single_mut();
if right_trigger > 0.8 {
*right_state = XRInteractorState::Selecting;
} else {
*right_state = XRInteractorState::Idle;
}
}
#[derive(Component)]
pub struct Grabbable;
pub fn update_grabbables(
mut events: EventReader<InteractionEvent>,
mut grabbable_query: Query<(
Entity,
&mut Transform,
With<Grabbable>,
Without<XRDirectInteractor>,
Option<&mut RigidBody>,
)>,
mut interactor_query: Query<(
&GlobalTransform,
&XRInteractorState,
&mut XRSelection,
Without<Grabbable>,
)>,
) {
//so basically the idea is to try all the events?
for event in events.read() {
// info!("some event");
match grabbable_query.get_mut(event.interactable) {
Ok(mut grabbable_transform) => {
// info!("we got a grabbable");
//now we need the location of our interactor
match interactor_query.get_mut(event.interactor) {
Ok(mut interactor_transform) => {
match *interactor_transform.2 {
XRSelection::Empty => {
match interactor_transform.1 {
XRInteractorState::Idle => match grabbable_transform.4 {
Some(mut thing) => {
*thing = RigidBody::Dynamic;
*interactor_transform.2 = XRSelection::Empty;
}
None => (),
},
XRInteractorState::Selecting => {
// info!("its a direct interactor?");
match grabbable_transform.4 {
Some(mut thing) => {
*thing = RigidBody::KinematicPositionBased;
*interactor_transform.2 =
XRSelection::Full(grabbable_transform.0);
}
None => (),
}
*grabbable_transform.1 =
interactor_transform.0.compute_transform();
}
}
}
XRSelection::Full(ent) => {
info!("nah bro we holding something");
match grabbable_transform.0 == ent {
true => {
*grabbable_transform.1 =
interactor_transform.0.compute_transform();
}
false => {}
}
match interactor_transform.1 {
XRInteractorState::Idle => {
*interactor_transform.2 = XRSelection::Empty
}
XRInteractorState::Selecting => {}
}
}
}
}
Err(_) => {
// info!("not a direct interactor")
}
}
}
Err(_) => {
// info!("not a grabbable?")
}
}
}
}

4
examples/xr.rs
View File

@@ -3,10 +3,10 @@ use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin};
use bevy::prelude::*;
use bevy::transform::components::Transform;
use bevy_oxr::input::XrInput;
use bevy_oxr::resources::{XrFrameState, XrInstance, XrSession};
use bevy_oxr::resources::{XrFrameState, XrSession};
use bevy_oxr::xr_input::actions::XrActionSets;
use bevy_oxr::xr_input::hand::{HandInputDebugRenderer, OpenXrHandInput};
use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput};
use bevy_oxr::xr_input::interactions::{
draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
update_interactable_states, InteractionEvent, Touched, XRDirectInteractor, XRInteractable,

6
src/graphics/vulkan.rs
View File

@@ -20,6 +20,8 @@ use crate::VIEW_TYPE;
pub fn initialize_xr_graphics(
window: Option<RawHandleWrapper>,
// Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
// any extension at some point
) -> anyhow::Result<(
RenderDevice,
RenderQueue,
@@ -37,6 +39,8 @@ pub fn initialize_xr_graphics(
XrInput,
XrViews,
XrFrameState,
// Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
// any extension at some point
)> {
use wgpu_hal::{api::Vulkan as V, Api};
@@ -405,6 +409,8 @@ pub fn initialize_xr_graphics(
should_render: true,
})
.into(),
// Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
// any extension at some point
))
}

35
src/lib.rs
View File

@@ -6,6 +6,8 @@ pub mod xr_input;
use std::sync::{Arc, Mutex};
use crate::xr_input::hands::hand_tracking::DisableHandTracking;
use crate::xr_input::oculus_touch::ActionSets;
use bevy::app::PluginGroupBuilder;
use bevy::ecs::system::SystemState;
use bevy::prelude::*;
@@ -18,22 +20,26 @@ use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper};
use input::XrInput;
use openxr as xr;
use resources::*;
use xr::FormFactor;
use xr_input::controllers::XrControllerType;
use xr_input::hand::HandInputSource;
use xr_input::handtracking::HandTrackingTracker;
use xr_input::hands::emulated::HandEmulationPlugin;
use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin};
use xr_input::OpenXrInput;
use crate::xr_input::oculus_touch::ActionSets;
const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
pub const LEFT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(1208214591);
pub const RIGHT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(3383858418);
/// Adds OpenXR support to an App
#[derive(Default)]
pub struct OpenXrPlugin;
impl Default for OpenXrPlugin {
fn default() -> Self {
OpenXrPlugin
}
}
#[derive(Resource)]
pub struct FutureXrResources(
pub Arc<
@@ -82,6 +88,7 @@ impl Plugin for OpenXrPlugin {
views,
frame_state,
) = graphics::initialize_xr_graphics(primary_window).unwrap();
// std::thread::sleep(Duration::from_secs(5));
debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
debug!("Configured wgpu adapter Features: {:#?}", device.features());
let mut future_xr_resources_inner = future_xr_resources_wrapper.lock().unwrap();
@@ -147,12 +154,18 @@ impl Plugin for OpenXrPlugin {
.insert_resource(views.clone())
.insert_resource(frame_state.clone())
.insert_resource(action_sets.clone());
let hands = xr_instance.exts().ext_hand_tracking.is_some();
let hands = xr_instance.exts().ext_hand_tracking.is_some()
&& xr_instance
.supports_hand_tracking(
xr_instance
.system(FormFactor::HEAD_MOUNTED_DISPLAY)
.unwrap(),
)
.is_ok_and(|v| v);
if hands {
app.insert_resource(HandTrackingTracker::new(&session).unwrap());
app.insert_resource(HandInputSource::OpenXr);
app.insert_resource(HandTrackingData::new(&session).unwrap());
} else {
app.insert_resource(HandInputSource::Emulated);
app.insert_resource(DisableHandTracking::Both);
}
let (left, right) = swapchain.get_render_views();
@@ -208,8 +221,10 @@ impl PluginGroup for DefaultXrPlugins {
.build()
.disable::<RenderPlugin>()
.disable::<PipelinedRenderingPlugin>()
.add_before::<RenderPlugin, _>(OpenXrPlugin)
.add_before::<RenderPlugin, _>(OpenXrPlugin::default())
.add_after::<OpenXrPlugin, _>(OpenXrInput::new(XrControllerType::OculusTouch))
.add(HandEmulationPlugin)
.add(HandTrackingPlugin)
.set(WindowPlugin {
#[cfg(not(target_os = "android"))]
primary_window: Some(Window {

74
src/xr_input/debug_gizmos.rs
View File

@@ -1,11 +1,12 @@
use bevy::prelude::{
info, Color, Gizmos, GlobalTransform, Plugin, Quat, Query, Res, Transform, Update, Vec2, Vec3,
With, Without,
Color, Gizmos, GlobalTransform, Plugin, Quat, Query, Res, Transform, Update, Vec2, Vec3, With,
Without,
};
use bevy::log::info;
use crate::{
input::XrInput,
resources::{XrFrameState, XrInstance, XrSession},
resources::{XrFrameState, XrSession},
};
use crate::xr_input::{
@@ -15,9 +16,7 @@ use crate::xr_input::{
use super::{
actions::XrActionSets,
handtracking::{HandTrackingRef, HandTrackingTracker},
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
QuatConv,
};
/// add debug renderer for controllers
@@ -35,7 +34,6 @@ pub fn draw_gizmos(
oculus_controller: Res<OculusController>,
frame_state: Res<XrFrameState>,
xr_input: Res<XrInput>,
instance: Res<XrInstance>,
session: Res<XrSession>,
tracking_root_query: Query<(
&mut Transform,
@@ -55,49 +53,38 @@ pub fn draw_gizmos(
Without<OpenXRLeftController>,
Without<OpenXRTrackingRoot>,
)>,
hand_tracking: Option<Res<HandTrackingTracker>>,
action_sets: Res<XrActionSets>,
) {
if let Some(hand_tracking) = hand_tracking {
let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
if let Some(joints) = handtracking_ref.get_left_poses() {
for joint in joints {
let p = joint.pose.position;
let r = joint.pose.orientation;
let quat = r.to_quat();
let trans = Transform::from_rotation(quat);
gizmos.circle(
(p.x, p.y, p.z).into(),
trans.forward(),
joint.radius,
Color::ORANGE_RED,
);
}
} else {
info!("left_hand_poses returned None");
}
if let Some(joints) = handtracking_ref.get_right_poses() {
for joint in joints {
let p = joint.pose.position;
let r = joint.pose.orientation;
let quat = r.to_quat();
let trans = Transform::from_rotation(quat);
gizmos.circle(
(p.x, p.y, p.z).into(),
trans.forward(),
joint.radius,
Color::LIME_GREEN,
);
}
return;
}
}
// if let Some(hand_tracking) = hand_tracking {
// let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
// if let Some(joints) = handtracking_ref.get_poses(Hand::Left) {
// for joint in joints.inner() {
// let trans = Transform::from_rotation(joint.orientation);
// gizmos.circle(
// joint.position,
// trans.forward(),
// joint.radius,
// Color::ORANGE_RED,
// );
// }
// }
// if let Some(joints) = handtracking_ref.get_poses(Hand::Right) {
// for joint in joints.inner() {
// let trans = Transform::from_rotation(joint.orientation);
// gizmos.circle(
// joint.position,
// trans.forward(),
// joint.radius,
// Color::LIME_GREEN,
// );
// }
// return;
// }
// }
//lock frame
let frame_state = *frame_state.lock().unwrap();
//get controller
let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
//tracking root?
let mut tracking_transform = &Transform::IDENTITY;
let root = tracking_root_query.get_single();
match root {
Ok(position) => {
@@ -112,7 +99,6 @@ pub fn draw_gizmos(
0.2,
Color::RED,
);
tracking_transform = position.0;
}
Err(_) => info!("too many tracking roots"),
}

1197
src/xr_input/hand.rs
View File

File diff suppressed because it is too large Load Diff

286
src/xr_input/hands/common.rs Normal file
View File

@@ -0,0 +1,286 @@
use bevy::prelude::{
default, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, Plugin, PostUpdate,
Query, Resource, SpatialBundle, Startup, Transform,
};
use crate::xr_input::{Hand, trackers::OpenXRTracker};
use super::{HandBone, BoneTrackingStatus};
/// add debug renderer for controllers
#[derive(Default)]
pub struct OpenXrHandInput;
impl Plugin for OpenXrHandInput {
fn build(&self, app: &mut bevy::prelude::App) {
app.add_systems(Startup, spawn_hand_entities);
}
}
/// add debug renderer for controllers
#[derive(Default)]
pub struct HandInputDebugRenderer;
impl Plugin for HandInputDebugRenderer {
fn build(&self, app: &mut bevy::prelude::App) {
app.add_systems(PostUpdate, draw_hand_entities);
}
}
#[derive(Resource, Default, Clone, Copy)]
pub struct HandsResource {
pub left: HandResource,
pub right: HandResource,
}
#[derive(Clone, Copy)]
pub struct HandResource {
pub palm: Entity,
pub wrist: Entity,
pub thumb: ThumbResource,
pub index: IndexResource,
pub middle: MiddleResource,
pub ring: RingResource,
pub little: LittleResource,
}
impl Default for HandResource {
fn default() -> Self {
Self {
palm: Entity::PLACEHOLDER,
wrist: Entity::PLACEHOLDER,
thumb: Default::default(),
index: Default::default(),
middle: Default::default(),
ring: Default::default(),
little: Default::default(),
}
}
}
#[derive(Clone, Copy)]
pub struct ThumbResource {
pub metacarpal: Entity,
pub proximal: Entity,
pub distal: Entity,
pub tip: Entity,
}
impl Default for ThumbResource {
fn default() -> Self {
Self {
metacarpal: Entity::PLACEHOLDER,
proximal: Entity::PLACEHOLDER,
distal: Entity::PLACEHOLDER,
tip: Entity::PLACEHOLDER,
}
}
}
#[derive(Clone, Copy)]
pub struct IndexResource {
pub metacarpal: Entity,
pub proximal: Entity,
pub intermediate: Entity,
pub distal: Entity,
pub tip: Entity,
}
impl Default for IndexResource {
fn default() -> Self {
Self {
metacarpal: Entity::PLACEHOLDER,
proximal: Entity::PLACEHOLDER,
intermediate: Entity::PLACEHOLDER,
distal: Entity::PLACEHOLDER,
tip: Entity::PLACEHOLDER,
}
}
}
#[derive(Clone, Copy)]
pub struct MiddleResource {
pub metacarpal: Entity,
pub proximal: Entity,
pub intermediate: Entity,
pub distal: Entity,
pub tip: Entity,
}
impl Default for MiddleResource {
fn default() -> Self {
Self {
metacarpal: Entity::PLACEHOLDER,
proximal: Entity::PLACEHOLDER,
intermediate: Entity::PLACEHOLDER,
distal: Entity::PLACEHOLDER,
tip: Entity::PLACEHOLDER,
}
}
}
#[derive(Clone, Copy)]
pub struct RingResource {
pub metacarpal: Entity,
pub proximal: Entity,
pub intermediate: Entity,
pub distal: Entity,
pub tip: Entity,
}
impl Default for RingResource {
fn default() -> Self {
Self {
metacarpal: Entity::PLACEHOLDER,
proximal: Entity::PLACEHOLDER,
intermediate: Entity::PLACEHOLDER,
distal: Entity::PLACEHOLDER,
tip: Entity::PLACEHOLDER,
}
}
}
#[derive(Clone, Copy)]
pub struct LittleResource {
pub metacarpal: Entity,
pub proximal: Entity,
pub intermediate: Entity,
pub distal: Entity,
pub tip: Entity,
}
impl Default for LittleResource {
fn default() -> Self {
Self {
metacarpal: Entity::PLACEHOLDER,
proximal: Entity::PLACEHOLDER,
intermediate: Entity::PLACEHOLDER,
distal: Entity::PLACEHOLDER,
tip: Entity::PLACEHOLDER,
}
}
}
pub fn spawn_hand_entities(mut commands: Commands) {
let hands = [Hand::Left, Hand::Right];
let bones = HandBone::get_all_bones();
//hand resource
let mut hand_resource = HandsResource { ..default() };
for hand in hands.iter() {
for bone in bones.iter() {
let boneid = commands
.spawn((
SpatialBundle::default(),
bone.clone(),
OpenXRTracker,
hand.clone(),
BoneTrackingStatus::Emulated,
HandBoneRadius(0.1),
))
.id();
match hand {
Hand::Left => match bone {
HandBone::Palm => hand_resource.left.palm = boneid,
HandBone::Wrist => hand_resource.left.wrist = boneid,
HandBone::ThumbMetacarpal => hand_resource.left.thumb.metacarpal = boneid,
HandBone::ThumbProximal => hand_resource.left.thumb.proximal = boneid,
HandBone::ThumbDistal => hand_resource.left.thumb.distal = boneid,
HandBone::ThumbTip => hand_resource.left.thumb.tip = boneid,
HandBone::IndexMetacarpal => hand_resource.left.index.metacarpal = boneid,
HandBone::IndexProximal => hand_resource.left.index.proximal = boneid,
HandBone::IndexIntermediate => hand_resource.left.index.intermediate = boneid,
HandBone::IndexDistal => hand_resource.left.index.distal = boneid,
HandBone::IndexTip => hand_resource.left.index.tip = boneid,
HandBone::MiddleMetacarpal => hand_resource.left.middle.metacarpal = boneid,
HandBone::MiddleProximal => hand_resource.left.middle.proximal = boneid,
HandBone::MiddleIntermediate => hand_resource.left.middle.intermediate = boneid,
HandBone::MiddleDistal => hand_resource.left.middle.distal = boneid,
HandBone::MiddleTip => hand_resource.left.middle.tip = boneid,
HandBone::RingMetacarpal => hand_resource.left.ring.metacarpal = boneid,
HandBone::RingProximal => hand_resource.left.ring.proximal = boneid,
HandBone::RingIntermediate => hand_resource.left.ring.intermediate = boneid,
HandBone::RingDistal => hand_resource.left.ring.distal = boneid,
HandBone::RingTip => hand_resource.left.ring.tip = boneid,
HandBone::LittleMetacarpal => hand_resource.left.little.metacarpal = boneid,
HandBone::LittleProximal => hand_resource.left.little.proximal = boneid,
HandBone::LittleIntermediate => hand_resource.left.little.intermediate = boneid,
HandBone::LittleDistal => hand_resource.left.little.distal = boneid,
HandBone::LittleTip => hand_resource.left.little.tip = boneid,
},
Hand::Right => match bone {
HandBone::Palm => hand_resource.right.palm = boneid,
HandBone::Wrist => hand_resource.right.wrist = boneid,
HandBone::ThumbMetacarpal => hand_resource.right.thumb.metacarpal = boneid,
HandBone::ThumbProximal => hand_resource.right.thumb.proximal = boneid,
HandBone::ThumbDistal => hand_resource.right.thumb.distal = boneid,
HandBone::ThumbTip => hand_resource.right.thumb.tip = boneid,
HandBone::IndexMetacarpal => hand_resource.right.index.metacarpal = boneid,
HandBone::IndexProximal => hand_resource.right.index.proximal = boneid,
HandBone::IndexIntermediate => hand_resource.right.index.intermediate = boneid,
HandBone::IndexDistal => hand_resource.right.index.distal = boneid,
HandBone::IndexTip => hand_resource.right.index.tip = boneid,
HandBone::MiddleMetacarpal => hand_resource.right.middle.metacarpal = boneid,
HandBone::MiddleProximal => hand_resource.right.middle.proximal = boneid,
HandBone::MiddleIntermediate => {
hand_resource.right.middle.intermediate = boneid
}
HandBone::MiddleDistal => hand_resource.right.middle.distal = boneid,
HandBone::MiddleTip => hand_resource.right.middle.tip = boneid,
HandBone::RingMetacarpal => hand_resource.right.ring.metacarpal = boneid,
HandBone::RingProximal => hand_resource.right.ring.proximal = boneid,
HandBone::RingIntermediate => hand_resource.right.ring.intermediate = boneid,
HandBone::RingDistal => hand_resource.right.ring.distal = boneid,
HandBone::RingTip => hand_resource.right.ring.tip = boneid,
HandBone::LittleMetacarpal => hand_resource.right.little.metacarpal = boneid,
HandBone::LittleProximal => hand_resource.right.little.proximal = boneid,
HandBone::LittleIntermediate => {
hand_resource.right.little.intermediate = boneid
}
HandBone::LittleDistal => hand_resource.right.little.distal = boneid,
HandBone::LittleTip => hand_resource.right.little.tip = boneid,
},
}
}
}
commands.insert_resource(hand_resource);
}
#[derive(Debug, Component, DerefMut, Deref)]
pub struct HandBoneRadius(pub f32);
pub fn draw_hand_entities(
mut gizmos: Gizmos,
query: Query<(&Transform, &HandBone, &HandBoneRadius)>,
) {
for (transform, hand_bone, hand_bone_radius) in query.iter() {
let (_, color) = get_bone_gizmo_style(hand_bone);
gizmos.sphere(
transform.translation,
transform.rotation,
hand_bone_radius.0,
color,
);
}
}
pub(crate) fn get_bone_gizmo_style(hand_bone: &HandBone) -> (f32, Color) {
match hand_bone {
HandBone::Palm => (0.01, Color::WHITE),
HandBone::Wrist => (0.01, Color::GRAY),
HandBone::ThumbMetacarpal => (0.01, Color::RED),
HandBone::ThumbProximal => (0.008, Color::RED),
HandBone::ThumbDistal => (0.006, Color::RED),
HandBone::ThumbTip => (0.004, Color::RED),
HandBone::IndexMetacarpal => (0.01, Color::ORANGE),
HandBone::IndexProximal => (0.008, Color::ORANGE),
HandBone::IndexIntermediate => (0.006, Color::ORANGE),
HandBone::IndexDistal => (0.004, Color::ORANGE),
HandBone::IndexTip => (0.002, Color::ORANGE),
HandBone::MiddleMetacarpal => (0.01, Color::YELLOW),
HandBone::MiddleProximal => (0.008, Color::YELLOW),
HandBone::MiddleIntermediate => (0.006, Color::YELLOW),
HandBone::MiddleDistal => (0.004, Color::YELLOW),
HandBone::MiddleTip => (0.002, Color::YELLOW),
HandBone::RingMetacarpal => (0.01, Color::GREEN),
HandBone::RingProximal => (0.008, Color::GREEN),
HandBone::RingIntermediate => (0.006, Color::GREEN),
HandBone::RingDistal => (0.004, Color::GREEN),
HandBone::RingTip => (0.002, Color::GREEN),
HandBone::LittleMetacarpal => (0.01, Color::BLUE),
HandBone::LittleProximal => (0.008, Color::BLUE),
HandBone::LittleIntermediate => (0.006, Color::BLUE),
HandBone::LittleDistal => (0.004, Color::BLUE),
HandBone::LittleTip => (0.002, Color::BLUE),
}
}

528
src/xr_input/hands/emulated.rs Normal file
View File

@@ -0,0 +1,528 @@
use std::f32::consts::PI;
use bevy::prelude::*;
use openxr::{ActionTy, HandJoint};
use super::common::{get_bone_gizmo_style, HandBoneRadius};
use crate::{
resources::{XrInstance, XrSession},
xr_input::{
actions::{
ActionHandednes, ActionType, SetupActionSet, SetupActionSets, XrActionSets, XrBinding,
},
hand_poses::get_simulated_open_hand_transforms,
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
Hand,
},
};
use super::{BoneTrackingStatus, HandBone};
pub enum TouchValue<T: ActionTy> {
None,
Touched(T),
}
pub struct HandEmulationPlugin;
impl Plugin for HandEmulationPlugin {
fn build(&self, app: &mut App) {
app.add_systems(PreUpdate, update_hand_skeleton_from_emulated);
app.add_systems(Startup, setup_hand_emulation_action_set);
}
}
const HAND_ACTION_SET: &str = "hand_pose_approx";
fn setup_hand_emulation_action_set(mut action_sets: ResMut<SetupActionSets>) {
let mut action_set = action_sets.add_action_set(HAND_ACTION_SET, "Hand Pose Approximaiton", 0);
action_set.new_action(
"thumb_touch",
"Thumb Touched",
ActionType::Bool,
ActionHandednes::Double,
);
action_set.new_action(
"thumb_x",
"Thumb X",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"thumb_y",
"Thumb Y",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"index_touch",
"Index Finger Touched",
ActionType::Bool,
ActionHandednes::Double,
);
action_set.new_action(
"index_value",
"Index Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"middle_value",
"Middle Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"ring_value",
"Ring Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
action_set.new_action(
"little_value",
"Little Finger Pull",
ActionType::F32,
ActionHandednes::Double,
);
suggest_oculus_touch_profile(&mut action_set);
}
pub struct EmulatedHandPoseData {}
fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) {
action_set.suggest_binding(
"/interaction_profiles/oculus/touch_controller",
&[
XrBinding::new("thumb_x", "/user/hand/left/input/thumbstick/x"),
XrBinding::new("thumb_x", "/user/hand/right/input/thumbstick/x"),
XrBinding::new("thumb_y", "/user/hand/left/input/thumbstick/y"),
XrBinding::new("thumb_y", "/user/hand/right/input/thumbstick/y"),
XrBinding::new("thumb_touch", "/user/hand/left/input/thumbstick/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/thumbstick/touch"),
XrBinding::new("thumb_touch", "/user/hand/left/input/x/touch"),
XrBinding::new("thumb_touch", "/user/hand/left/input/y/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/a/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/b/touch"),
XrBinding::new("thumb_touch", "/user/hand/left/input/thumbrest/touch"),
XrBinding::new("thumb_touch", "/user/hand/right/input/thumbrest/touch"),
XrBinding::new("index_touch", "/user/hand/left/input/trigger/touch"),
XrBinding::new("index_value", "/user/hand/left/input/trigger/value"),
XrBinding::new("index_touch", "/user/hand/right/input/trigger/touch"),
XrBinding::new("index_value", "/user/hand/right/input/trigger/value"),
XrBinding::new("middle_value", "/user/hand/left/input/squeeze/value"),
XrBinding::new("middle_value", "/user/hand/right/input/squeeze/value"),
XrBinding::new("ring_value", "/user/hand/left/input/squeeze/value"),
XrBinding::new("ring_value", "/user/hand/right/input/squeeze/value"),
XrBinding::new("little_value", "/user/hand/left/input/squeeze/value"),
XrBinding::new("little_value", "/user/hand/right/input/squeeze/value"),
],
);
}
pub(crate) fn update_hand_skeleton_from_emulated(
session: Res<XrSession>,
instance: Res<XrInstance>,
action_sets: Res<XrActionSets>,
left_controller_transform: Query<&Transform, With<OpenXRLeftController>>,
right_controller_transform: Query<&Transform, With<OpenXRRightController>>,
tracking_root_transform: Query<&Transform, With<OpenXRTrackingRoot>>,
mut bones: Query<
(
&mut Transform,
&HandBone,
&Hand,
&BoneTrackingStatus,
&mut HandBoneRadius,
),
(
Without<OpenXRLeftController>,
Without<OpenXRRightController>,
Without<OpenXRTrackingRoot>,
),
>,
) {
let mut data: [[Transform; 26]; 2] = [[Transform::default(); 26]; 2];
for (subaction_path, hand) in [
(
instance.string_to_path("/user/hand/left").unwrap(),
Hand::Left,
),
(
instance.string_to_path("/user/hand/right").unwrap(),
Hand::Right,
),
] {
let thumb_curl = match action_sets
.get_action_bool(HAND_ACTION_SET, "thumb_touch")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state
{
true => 1.0,
false => 0.0,
};
let index_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "index_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
let middle_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "middle_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
let ring_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "ring_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
let little_curl = action_sets
.get_action_f32(HAND_ACTION_SET, "little_value")
.unwrap()
.state(&session, subaction_path)
.unwrap()
.current_state;
data[match hand {
Hand::Left => 0,
Hand::Right => 1,
}] = update_hand_bones_emulated(
match hand {
Hand::Left => left_controller_transform.single(),
Hand::Right => right_controller_transform.single(),
},
hand,
thumb_curl,
index_curl,
middle_curl,
ring_curl,
little_curl,
);
}
let trt = tracking_root_transform.single();
for (mut t, bone, hand, status, mut radius) in bones.iter_mut() {
match status {
BoneTrackingStatus::Emulated => {}
BoneTrackingStatus::Tracked => continue,
}
radius.0 = get_bone_gizmo_style(bone).0;
*t = data[match hand {
Hand::Left => 0,
Hand::Right => 1,
}][bone.get_index_from_bone()];
*t = t.with_scale(trt.scale);
*t = t.with_rotation(trt.rotation * t.rotation);
*t = t.with_translation(trt.transform_point(t.translation));
}
}
pub fn update_hand_bones_emulated(
controller_transform: &Transform,
hand: Hand,
thumb_curl: f32,
index_curl: f32,
middle_curl: f32,
ring_curl: f32,
little_curl: f32,
) -> [Transform; 26] {
let left_hand_rot = Quat::from_rotation_y(PI);
let hand_translation: Vec3 = controller_transform.translation;
let controller_quat: Quat = match hand {
Hand::Left => controller_transform.rotation.mul_quat(left_hand_rot),
Hand::Right => controller_transform.rotation,
};
let splay_direction = match hand {
Hand::Left => -1.0,
Hand::Right => 1.0,
};
//lets make a structure to hold our calculated transforms for now
let mut calc_transforms = [Transform::default(); 26];
//get palm quat
let y = Quat::from_rotation_y(-90.0 * PI / 180.0);
let x = Quat::from_rotation_x(-90.0 * PI / 180.0);
let palm_quat = controller_quat.mul_quat(y).mul_quat(x);
//get simulated bones
let hand_transform_array: [Transform; 26] = get_simulated_open_hand_transforms(hand);
//palm
let palm = hand_transform_array[HandJoint::PALM];
calc_transforms[HandJoint::PALM] = Transform {
translation: hand_translation + palm.translation,
..default()
};
//wrist
let wrist = hand_transform_array[HandJoint::WRIST];
calc_transforms[HandJoint::WRIST] = Transform {
translation: hand_translation + palm.translation + palm_quat.mul_vec3(wrist.translation),
..default()
};
//thumb
let thumb_joints = [
HandJoint::THUMB_METACARPAL,
HandJoint::THUMB_PROXIMAL,
HandJoint::THUMB_DISTAL,
HandJoint::THUMB_TIP,
];
let mut prior_start: Option<Vec3> = None;
let mut prior_quat: Option<Quat> = None;
let mut prior_vector: Option<Vec3> = None;
let splay = Quat::from_rotation_y(splay_direction * 30.0 * PI / 180.0);
let huh = Quat::from_rotation_x(-35.0 * PI / 180.0);
let splay_quat = palm_quat.mul_quat(huh).mul_quat(splay);
for bone in thumb_joints.iter() {
match prior_start {
Some(start) => {
let curl_angle: f32 = get_bone_curl_angle(*bone, thumb_curl);
let tp_lrot = Quat::from_rotation_y(splay_direction * curl_angle * PI / 180.0);
let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
let thumb_prox = hand_transform_array[*bone];
let tp_start = start + prior_vector.unwrap();
let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
prior_start = Some(tp_start);
prior_quat = Some(tp_quat);
prior_vector = Some(tp_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tp_start + tp_vector,
..default()
};
}
None => {
let thumb_meta = hand_transform_array[*bone];
let tm_start = hand_translation
+ palm_quat.mul_vec3(palm.translation)
+ palm_quat.mul_vec3(wrist.translation);
let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
prior_start = Some(tm_start);
prior_quat = Some(splay_quat);
prior_vector = Some(tm_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tm_start + tm_vector,
..default()
};
}
}
}
//index
let thumb_joints = [
HandJoint::INDEX_METACARPAL,
HandJoint::INDEX_PROXIMAL,
HandJoint::INDEX_INTERMEDIATE,
HandJoint::INDEX_DISTAL,
HandJoint::INDEX_TIP,
];
let mut prior_start: Option<Vec3> = None;
let mut prior_quat: Option<Quat> = None;
let mut prior_vector: Option<Vec3> = None;
let splay = Quat::from_rotation_y(splay_direction * 10.0 * PI / 180.0);
let splay_quat = palm_quat.mul_quat(splay);
for bone in thumb_joints.iter() {
match prior_start {
Some(start) => {
let curl_angle: f32 = get_bone_curl_angle(*bone, index_curl);
let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
let thumb_prox = hand_transform_array[*bone];
let tp_start = start + prior_vector.unwrap();
let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
prior_start = Some(tp_start);
prior_quat = Some(tp_quat);
prior_vector = Some(tp_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tp_start + tp_vector,
..default()
};
}
None => {
let thumb_meta = hand_transform_array[*bone];
let tm_start = hand_translation
+ palm_quat.mul_vec3(palm.translation)
+ palm_quat.mul_vec3(wrist.translation);
let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
prior_start = Some(tm_start);
prior_quat = Some(splay_quat);
prior_vector = Some(tm_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tm_start + tm_vector,
..default()
};
}
}
}
//middle
let thumb_joints = [
HandJoint::MIDDLE_METACARPAL,
HandJoint::MIDDLE_PROXIMAL,
HandJoint::MIDDLE_INTERMEDIATE,
HandJoint::MIDDLE_DISTAL,
HandJoint::MIDDLE_TIP,
];
let mut prior_start: Option<Vec3> = None;
let mut prior_quat: Option<Quat> = None;
let mut prior_vector: Option<Vec3> = None;
let splay = Quat::from_rotation_y(splay_direction * 0.0 * PI / 180.0);
let splay_quat = palm_quat.mul_quat(splay);
for bone in thumb_joints.iter() {
match prior_start {
Some(start) => {
let curl_angle: f32 = get_bone_curl_angle(*bone, middle_curl);
let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
let thumb_prox = hand_transform_array[*bone];
let tp_start = start + prior_vector.unwrap();
let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
prior_start = Some(tp_start);
prior_quat = Some(tp_quat);
prior_vector = Some(tp_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tp_start + tp_vector,
..default()
};
}
None => {
let thumb_meta = hand_transform_array[*bone];
let tm_start = hand_translation
+ palm_quat.mul_vec3(palm.translation)
+ palm_quat.mul_vec3(wrist.translation);
let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
prior_start = Some(tm_start);
prior_quat = Some(splay_quat);
prior_vector = Some(tm_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tm_start + tm_vector,
..default()
};
}
}
}
//ring
let thumb_joints = [
HandJoint::RING_METACARPAL,
HandJoint::RING_PROXIMAL,
HandJoint::RING_INTERMEDIATE,
HandJoint::RING_DISTAL,
HandJoint::RING_TIP,
];
let mut prior_start: Option<Vec3> = None;
let mut prior_quat: Option<Quat> = None;
let mut prior_vector: Option<Vec3> = None;
let splay = Quat::from_rotation_y(splay_direction * -10.0 * PI / 180.0);
let splay_quat = palm_quat.mul_quat(splay);
for bone in thumb_joints.iter() {
match prior_start {
Some(start) => {
let curl_angle: f32 = get_bone_curl_angle(*bone, ring_curl);
let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
let thumb_prox = hand_transform_array[*bone];
let tp_start = start + prior_vector.unwrap();
let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
prior_start = Some(tp_start);
prior_quat = Some(tp_quat);
prior_vector = Some(tp_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tp_start + tp_vector,
..default()
};
}
None => {
let thumb_meta = hand_transform_array[*bone];
let tm_start = hand_translation
+ palm_quat.mul_vec3(palm.translation)
+ palm_quat.mul_vec3(wrist.translation);
let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
prior_start = Some(tm_start);
prior_quat = Some(splay_quat);
prior_vector = Some(tm_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tm_start + tm_vector,
..default()
};
}
}
}
//little
let thumb_joints = [
HandJoint::LITTLE_METACARPAL,
HandJoint::LITTLE_PROXIMAL,
HandJoint::LITTLE_INTERMEDIATE,
HandJoint::LITTLE_DISTAL,
HandJoint::LITTLE_TIP,
];
let mut prior_start: Option<Vec3> = None;
let mut prior_quat: Option<Quat> = None;
let mut prior_vector: Option<Vec3> = None;
let splay = Quat::from_rotation_y(splay_direction * -20.0 * PI / 180.0);
let splay_quat = palm_quat.mul_quat(splay);
for bone in thumb_joints.iter() {
match prior_start {
Some(start) => {
let curl_angle: f32 = get_bone_curl_angle(*bone, little_curl);
let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
let thumb_prox = hand_transform_array[*bone];
let tp_start = start + prior_vector.unwrap();
let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
prior_start = Some(tp_start);
prior_quat = Some(tp_quat);
prior_vector = Some(tp_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tp_start + tp_vector,
..default()
};
}
None => {
let thumb_meta = hand_transform_array[*bone];
let tm_start = hand_translation
+ palm_quat.mul_vec3(palm.translation)
+ palm_quat.mul_vec3(wrist.translation);
let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
prior_start = Some(tm_start);
prior_quat = Some(splay_quat);
prior_vector = Some(tm_vector);
//store it
calc_transforms[*bone] = Transform {
translation: tm_start + tm_vector,
..default()
};
}
}
}
calc_transforms
}
fn get_bone_curl_angle(bone: HandJoint, curl: f32) -> f32 {
let mul: f32 = match bone {
HandJoint::INDEX_PROXIMAL => 0.0,
HandJoint::MIDDLE_PROXIMAL => 0.0,
HandJoint::RING_PROXIMAL => 0.0,
HandJoint::LITTLE_PROXIMAL => 0.0,
HandJoint::THUMB_PROXIMAL => 0.0,
HandJoint::THUMB_TIP => 0.1,
HandJoint::THUMB_DISTAL => 0.1,
HandJoint::THUMB_METACARPAL => 0.1,
_ => 1.0,
};
let curl_angle = -((mul * curl * 80.0) + 5.0);
return curl_angle;
}

211
src/xr_input/hands/hand_tracking.rs Normal file
View File

@@ -0,0 +1,211 @@
use bevy::prelude::*;
use openxr::{HandTracker, Result, SpaceLocationFlags};
use crate::{
input::XrInput,
resources::{XrFrameState, XrSession},
xr_input::{
hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv,
Vec3Conv,
},
};
use super::common::HandBoneRadius;
use super::BoneTrackingStatus;
#[derive(Resource, PartialEq)]
pub enum DisableHandTracking {
OnlyLeft,
OnlyRight,
Both,
}
pub struct HandTrackingPlugin;
#[derive(Resource)]
pub struct HandTrackingData {
left_hand: HandTracker,
right_hand: HandTracker,
}
impl HandTrackingData {
pub fn new(session: &XrSession) -> Result<HandTrackingData> {
let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
Ok(HandTrackingData {
left_hand: left,
right_hand: right,
})
}
pub fn get_ref<'a>(
&'a self,
input: &'a XrInput,
frame_state: &'a XrFrameState,
) -> HandTrackingRef<'a> {
HandTrackingRef {
tracking: self,
input,
frame_state,
}
}
}
pub struct HandTrackingRef<'a> {
tracking: &'a HandTrackingData,
input: &'a XrInput,
frame_state: &'a XrFrameState,
}
#[derive(Debug)]
pub struct HandJoint {
pub position: Vec3,
pub position_valid: bool,
pub position_tracked: bool,
pub orientation: Quat,
pub orientation_valid: bool,
pub orientation_tracked: bool,
pub radius: f32,
}
pub struct HandJoints {
inner: [HandJoint; 26],
}
impl HandJoints {
pub fn inner(&self) -> &[HandJoint; 26] {
&self.inner
}
}
impl HandJoints {
pub fn get_joint(&self, bone: HandBone) -> &HandJoint {
&self.inner[bone.get_index_from_bone()]
}
}
impl<'a> HandTrackingRef<'a> {
pub fn get_poses(&self, side: Hand) -> Option<HandJoints> {
self.input
.stage
.locate_hand_joints(
match side {
Hand::Left => &self.tracking.left_hand,
Hand::Right => &self.tracking.right_hand,
},
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
.map(|joints| {
joints
.into_iter()
.map(|joint| HandJoint {
position: joint.pose.position.to_vec3(),
orientation: joint.pose.orientation.to_quat(),
position_valid: joint
.location_flags
.contains(SpaceLocationFlags::POSITION_VALID),
position_tracked: joint
.location_flags
.contains(SpaceLocationFlags::POSITION_TRACKED),
orientation_valid: joint
.location_flags
.contains(SpaceLocationFlags::ORIENTATION_VALID),
orientation_tracked: joint
.location_flags
.contains(SpaceLocationFlags::ORIENTATION_TRACKED),
radius: joint.radius,
})
.collect::<Vec<HandJoint>>()
.try_into()
.unwrap()
})
.map(|joints| HandJoints { inner: joints })
}
}
impl Plugin for HandTrackingPlugin {
fn build(&self, app: &mut App) {
app.add_systems(
PreUpdate,
(
update_hand_bones.run_if(|dh: Option<Res<DisableHandTracking>>| {
!dh.is_some_and(|v| *v == DisableHandTracking::Both)
}),
update_tracking_state_on_disable,
),
);
}
}
fn update_tracking_state_on_disable(
mut is_off: Local<bool>,
disabled_tracking: Option<Res<DisableHandTracking>>,
mut tracking_states: Query<&mut BoneTrackingStatus>,
) {
if !*is_off
&& disabled_tracking
.as_ref()
.is_some_and(|t| **t == DisableHandTracking::Both)
{
tracking_states
.par_iter_mut()
.for_each(|mut state| *state = BoneTrackingStatus::Emulated);
}
*is_off = disabled_tracking
.as_ref()
.is_some_and(|t| **t == DisableHandTracking::Both);
}
pub fn update_hand_bones(
disabled_tracking: Option<Res<DisableHandTracking>>,
hand_tracking: Option<Res<HandTrackingData>>,
xr_input: Res<XrInput>,
xr_frame_state: Res<XrFrameState>,
root_query: Query<(&Transform, With<OpenXRTrackingRoot>, Without<HandBone>)>,
mut bones: Query<(
&mut Transform,
&Hand,
&HandBone,
&mut HandBoneRadius,
&mut BoneTrackingStatus,
)>,
) {
let hand_ref = match hand_tracking.as_ref() {
Some(h) => h.get_ref(&xr_input, &xr_frame_state),
None => {
warn!("No Handtracking data!");
return;
}
};
let (root_transform, _, _) = root_query.get_single().unwrap();
let left_hand_data = hand_ref.get_poses(Hand::Left);
let right_hand_data = hand_ref.get_poses(Hand::Right);
bones
.par_iter_mut()
.for_each(|(mut transform, hand, bone, mut radius, mut status)| {
match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) {
(Hand::Left, Some(DisableHandTracking::OnlyLeft)) => {
*status = BoneTrackingStatus::Emulated;
return;
}
(Hand::Right, Some(DisableHandTracking::OnlyRight)) => {
*status = BoneTrackingStatus::Emulated;
return;
}
_ => {}
}
let bone_data = match (hand, &left_hand_data, &right_hand_data) {
(Hand::Left, Some(data), _) => data.get_joint(*bone),
(Hand::Right, _, Some(data)) => data.get_joint(*bone),
_ => {
*status = BoneTrackingStatus::Emulated;
return;
}
};
if *status == BoneTrackingStatus::Emulated {
*status = BoneTrackingStatus::Tracked;
}
radius.0 = bone_data.radius;
*transform = transform
.with_translation(root_transform.transform_point(bone_data.position))
.with_rotation(root_transform.rotation * bone_data.orientation)
});
}

133
src/xr_input/hands/mod.rs Normal file
View File

@@ -0,0 +1,133 @@
use bevy::{app::PluginGroupBuilder, prelude::*};
use self::{emulated::HandEmulationPlugin, hand_tracking::HandTrackingPlugin};
pub mod emulated;
pub mod hand_tracking;
pub mod common;
pub struct XrHandPlugins;
impl PluginGroup for XrHandPlugins {
fn build(self) -> PluginGroupBuilder {
PluginGroupBuilder::start::<Self>()
.add(HandTrackingPlugin)
.add(HandEmulationPlugin)
.build()
}
}
#[derive(Component, Debug, Clone, Copy, PartialEq)]
pub enum BoneTrackingStatus {
Emulated,
Tracked,
}
#[derive(Component, Debug, Clone, Copy)]
pub enum HandBone {
Palm,
Wrist,
ThumbMetacarpal,
ThumbProximal,
ThumbDistal,
ThumbTip,
IndexMetacarpal,
IndexProximal,
IndexIntermediate,
IndexDistal,
IndexTip,
MiddleMetacarpal,
MiddleProximal,
MiddleIntermediate,
MiddleDistal,
MiddleTip,
RingMetacarpal,
RingProximal,
RingIntermediate,
RingDistal,
RingTip,
LittleMetacarpal,
LittleProximal,
LittleIntermediate,
LittleDistal,
LittleTip,
}
impl HandBone {
pub fn is_finger(&self) -> bool {
match &self {
HandBone::Wrist => false,
HandBone::Palm => false,
_ => true,
}
}
pub fn is_metacarpal(&self) -> bool {
match &self {
HandBone::ThumbMetacarpal => true,
HandBone::IndexMetacarpal => true,
HandBone::MiddleMetacarpal => true,
HandBone::RingMetacarpal => true,
HandBone::LittleTip => true,
_ => false,
}
}
pub const fn get_all_bones() -> [HandBone; 26] {
[
HandBone::Palm,
HandBone::Wrist,
HandBone::ThumbMetacarpal,
HandBone::ThumbProximal,
HandBone::ThumbDistal,
HandBone::ThumbTip,
HandBone::IndexMetacarpal,
HandBone::IndexProximal,
HandBone::IndexIntermediate,
HandBone::IndexDistal,
HandBone::IndexTip,
HandBone::MiddleMetacarpal,
HandBone::MiddleProximal,
HandBone::MiddleIntermediate,
HandBone::MiddleDistal,
HandBone::MiddleTip,
HandBone::RingMetacarpal,
HandBone::RingProximal,
HandBone::RingIntermediate,
HandBone::RingDistal,
HandBone::RingTip,
HandBone::LittleMetacarpal,
HandBone::LittleProximal,
HandBone::LittleIntermediate,
HandBone::LittleDistal,
HandBone::LittleTip,
]
}
pub fn get_index_from_bone(&self) -> usize {
match &self {
HandBone::Palm => 0,
HandBone::Wrist => 1,
HandBone::ThumbMetacarpal => 2,
HandBone::ThumbProximal => 3,
HandBone::ThumbDistal => 4,
HandBone::ThumbTip => 5,
HandBone::IndexMetacarpal => 6,
HandBone::IndexProximal => 7,
HandBone::IndexIntermediate => 8,
HandBone::IndexDistal => 9,
HandBone::IndexTip => 10,
HandBone::MiddleMetacarpal => 11,
HandBone::MiddleProximal => 12,
HandBone::MiddleIntermediate => 13,
HandBone::MiddleDistal => 14,
HandBone::MiddleTip => 15,
HandBone::RingMetacarpal => 16,
HandBone::RingProximal => 17,
HandBone::RingIntermediate => 18,
HandBone::RingDistal => 19,
HandBone::RingTip => 20,
HandBone::LittleMetacarpal => 21,
HandBone::LittleProximal => 22,
HandBone::LittleIntermediate => 23,
HandBone::LittleDistal => 24,
HandBone::LittleTip => 25,
}
}
}

84
src/xr_input/handtracking.rs
View File

@@ -1,84 +0,0 @@
use std::mem::MaybeUninit;
use bevy::prelude::*;
use openxr::{HandJointLocationEXT, HandTracker, Result};
use crate::{
input::XrInput,
resources::{XrFrameState, XrFrameWaiter, XrSession},
};
use super::hand::HandBone;
#[derive(Resource)]
pub struct HandTrackingTracker {
left_hand: HandTracker,
right_hand: HandTracker,
}
impl HandTrackingTracker {
pub fn new(session: &XrSession) -> Result<HandTrackingTracker> {
let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
Ok(HandTrackingTracker {
left_hand: left,
right_hand: right,
})
}
pub fn get_ref<'a>(
&'a self,
input: &'a XrInput,
frame_state: &'a XrFrameState,
) -> HandTrackingRef<'a> {
HandTrackingRef {
tracking: self,
input,
frame_state,
}
}
}
pub struct HandTrackingRef<'a> {
tracking: &'a HandTrackingTracker,
input: &'a XrInput,
frame_state: &'a XrFrameState,
}
// pub type HandJoints = [(HandJointLocationEXT, HandBone); 26];
impl<'a> HandTrackingRef<'a> {
pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT;26]> {
self.input
.stage
.locate_hand_joints(
&self.tracking.left_hand,
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
// .map(|joints| {
// joints
// .into_iter()
// .zip(HandBone::get_all_bones().into_iter())
// .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
// .try_into()
// .unwrap()
// })
}
pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT;26]> {
self.input
.stage
.locate_hand_joints(
&self.tracking.right_hand,
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
// .map(|joints| {
// joints
// .into_iter()
// .zip(HandBone::get_all_bones().into_iter())
// .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
// .try_into()
// .unwrap()
// })
}
}

5
src/xr_input/interactions.rs
View File

@@ -1,7 +1,8 @@
use std::f32::consts::PI;
use bevy::log::info;
use bevy::prelude::{
info, Color, Component, Entity, Event, EventReader, EventWriter, Gizmos, GlobalTransform, Quat,
Color, Component, Entity, Event, EventReader, EventWriter, Gizmos, GlobalTransform, Quat,
Query, Transform, Vec3, With, Without,
};
@@ -45,7 +46,7 @@ impl Default for XRInteractorState {
#[derive(Component)]
pub enum XRSelection {
Empty,
Full(Entity)
Full(Entity),
}
impl Default for XRSelection {
fn default() -> Self {

20
src/xr_input/mod.rs
View File

@@ -1,27 +1,28 @@
pub mod actions;
pub mod controllers;
pub mod debug_gizmos;
pub mod hand;
pub mod hand_poses;
pub mod handtracking;
pub mod hands;
pub mod interactions;
pub mod oculus_touch;
pub mod prototype_locomotion;
pub mod trackers;
pub mod xr_camera;
use crate::resources::XrSession;
use crate::resources::{XrInstance, XrSession};
use crate::xr_begin_frame;
use crate::xr_input::controllers::XrControllerType;
use crate::xr_input::oculus_touch::setup_oculus_controller;
use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle};
use bevy::app::{App, PostUpdate, Startup};
use bevy::log::warn;
use bevy::prelude::{BuildChildren, Component, IntoSystemConfigs};
use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource};
use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3};
use bevy::render::camera::CameraProjectionPlugin;
use bevy::render::view::{update_frusta, VisibilitySystems};
use bevy::transform::TransformSystem;
use bevy::utils::HashMap;
use openxr::Binding;
use self::actions::{setup_oxr_actions, OpenXrActionsPlugin};
use self::oculus_touch::{post_action_setup_oculus_controller, ActionSets};
@@ -75,6 +76,17 @@ impl Plugin for OpenXrInput {
}
}
#[derive(Deref, DerefMut, Resource)]
pub struct InteractionProfileBindings(pub HashMap<&'static str, Vec<Binding<'static>>>);
fn setup_binding_recommendations(
mut commands: Commands,
instance: Res<XrInstance>,
bindings: Res<InteractionProfileBindings>,
) {
commands.remove_resource::<InteractionProfileBindings>();
}
fn setup_xr_cameras(mut commands: Commands) {
//this needs to do the whole xr tracking volume not just cameras
//get the root?

89
src/xr_input/oculus_touch.rs
View File

@@ -1,17 +1,16 @@
use crate::input::XrInput;
use crate::resources::{XrInstance, XrSession};
use crate::xr_input::controllers::{Handed, Touchable};
use crate::xr_input::controllers::Handed;
use crate::xr_input::Hand;
use bevy::prelude::{Commands, Res, ResMut, Resource};
use openxr::{
Action, ActionSet, AnyGraphics, Binding, FrameState, Haptic, Instance, Path, Posef, Session,
Space, SpaceLocation, SpaceVelocity,
ActionSet, AnyGraphics, FrameState, Instance, Path, Posef, Session, Space, SpaceLocation,
SpaceVelocity,
};
use std::convert::identity;
use std::sync::OnceLock;
use super::actions::{ActionHandednes, XrActionSets, ActionType, SetupActionSets, XrBinding};
use super::actions::{ActionHandednes, ActionType, SetupActionSets, XrActionSets, XrBinding};
pub fn post_action_setup_oculus_controller(
action_sets: Res<XrActionSets>,
@@ -30,34 +29,18 @@ pub fn post_action_setup_oculus_controller(
.unwrap();
controller.grip_space = Some(Handed {
left: grip_action
.create_space(
s.clone(),
left_path,
Posef::IDENTITY,
)
.create_space(s.clone(), left_path, Posef::IDENTITY)
.unwrap(),
right: grip_action
.create_space(
s.clone(),
right_path,
Posef::IDENTITY,
)
.create_space(s.clone(), right_path, Posef::IDENTITY)
.unwrap(),
});
controller.aim_space = Some(Handed {
left: aim_action
.create_space(
s.clone(),
left_path,
Posef::IDENTITY,
)
.create_space(s.clone(), left_path, Posef::IDENTITY)
.unwrap(),
right: aim_action
.create_space(
s.clone(),
right_path,
Posef::IDENTITY,
)
.create_space(s.clone(), right_path, Posef::IDENTITY)
.unwrap(),
})
}
@@ -100,27 +83,51 @@ pub fn subaction_path(hand: Hand) -> Path {
impl OculusControllerRef<'_> {
pub fn grip_space(&self, hand: Hand) -> (SpaceLocation, SpaceVelocity) {
match hand {
Hand::Left => self.oculus_controller.grip_space.as_ref().unwrap().left.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
Hand::Right => self.oculus_controller.grip_space.as_ref().unwrap().right.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
Hand::Left => self
.oculus_controller
.grip_space
.as_ref()
.unwrap()
.left
.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
Hand::Right => self
.oculus_controller
.grip_space
.as_ref()
.unwrap()
.right
.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
}
.unwrap()
}
pub fn aim_space(&self, hand: Hand) -> (SpaceLocation, SpaceVelocity) {
match hand {
Hand::Left => self.oculus_controller.aim_space.as_ref().unwrap().left.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
Hand::Right => self.oculus_controller.aim_space.as_ref().unwrap().right.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
Hand::Left => self
.oculus_controller
.aim_space
.as_ref()
.unwrap()
.left
.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
Hand::Right => self
.oculus_controller
.aim_space
.as_ref()
.unwrap()
.right
.relate(
&self.xr_input.stage,
self.frame_state.predicted_display_time,
),
}
.unwrap()
}

6
src/xr_input/trackers.rs
View File

@@ -1,11 +1,11 @@
use bevy::log::info;
use bevy::prelude::{
info, Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With,
Without,
Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With, Without,
};
use crate::{
input::XrInput,
resources::{XrFrameState, XrInstance, XrSession},
resources::{XrFrameState, XrSession},
};
use super::{actions::XrActionSets, oculus_touch::OculusController, Hand, QuatConv, Vec3Conv};