Avii/bevy_oxr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Basic Handtracking Working

Browse Source
This commit is contained in:
Schmarni
2023-11-01 18:37:32 +01:00
parent c7e10bb537
commit 4c01eee827
10 changed files with 159 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -1,3 +1,3 @@
**/target
**/Cargo.lock
**/runtime_libs
**/runtime_libs/arm64-v8a/*

5
Cargo.toml
View File

@@ -4,7 +4,7 @@ version = "0.1.0"
edition = "2021"
[features]
default = ["linked"]
# default = ["linked"]
linked = ["openxr/linked", "openxr/static"]
[dependencies]
@@ -23,6 +23,9 @@ color-eyre = "0.6.2"
bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
# bevy_rapier3d = { git = "https://github.com/alexichepura/bevy_rapier", version = "0.22.0", branch = "bevy-012"}
[workspace]
members = ["examples/demo"]
[[example]]
name = "xr"
path = "examples/xr.rs"

3
examples/demo/Cargo.toml
View File

@@ -12,7 +12,8 @@ crate-type = ["cdylib","lib"]
bevy = { git = "https://github.com/bevyengine/bevy.git" }
# bevy = "0.11.3"
# default-features is false because it for some reason complains when trying to statically link openxr
bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"}
bevy_openxr = { path = "../../", default-features = false}
# bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"}
bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
color-eyre = "0.6.2"

8
examples/demo/manifest.yaml
View File

@@ -3,6 +3,14 @@ android:
- "runtime_libs"
manifest:
package: "org.bevyengine.demo_openxr_android"
uses_feature:
- name: "android.hardware.vr.headtracking"
required: true
- name: "oculus.software.handtracking"
required: false
- name: "com.oculus.experimental.enabled"
uses_permission:
- name: "com.oculus.permission.HAND_TRACKING"
application:
label: "Bevy Openxr Android"
theme: "@android:style/Theme.DeviceDefault.NoActionBar.Fullscreen"

4
src/graphics/vulkan.rs
View File

@@ -51,10 +51,14 @@ pub fn initialize_xr_graphics(
let mut enabled_extensions = xr::ExtensionSet::default();
enabled_extensions.khr_vulkan_enable2 = true;
enabled_extensions.khr_convert_timespec_time = true;
#[cfg(target_os = "android")]
{
enabled_extensions.khr_android_create_instance = true;
}
enabled_extensions.ext_hand_tracking = available_extensions.ext_hand_tracking;
// enabled_extensions.ext_hand_joints_motion_range = available_extensions.ext_hand_joints_motion_range;
let available_layers = xr_entry.enumerate_layers()?;
info!("available xr layers: {:#?}", available_layers);

4
src/lib.rs
View File

@@ -21,6 +21,7 @@ use input::XrInput;
use openxr as xr;
use resources::*;
use xr_input::controllers::XrControllerType;
use xr_input::handtracking::HandTrackingTracker;
use xr_input::OpenXrInput;
const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
@@ -145,7 +146,8 @@ impl Plugin for OpenXrPlugin {
.insert_resource(input.clone())
.insert_resource(views.clone())
.insert_resource(frame_state.clone())
.insert_resource(action_sets.clone());
.insert_resource(action_sets.clone())
.insert_resource(HandTrackingTracker::new(&session).unwrap());
let (left, right) = swapchain.get_render_views();
let left = ManualTextureView {

40
src/xr_input/debug_gizmos.rs
View File

@@ -13,7 +13,11 @@ use crate::xr_input::{
Hand,
};
use super::trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot};
use super::{
handtracking::{HandTrackingRef, HandTrackingTracker},
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
QuatConv,
};
/// add debug renderer for controllers
#[derive(Default)]
@@ -45,7 +49,41 @@ pub fn draw_gizmos(
Without<OpenXRLeftController>,
Without<OpenXRTrackingRoot>,
)>,
hand_tracking: Res<HandTrackingTracker>,
) {
let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
if let Some(joints) = handtracking_ref.get_left_poses() {
for joint in joints {
let p = joint.pose.position;
let r = joint.pose.orientation;
let quat = r.to_quat();
let trans = Transform::from_rotation(quat);
gizmos.circle(
(p.x, p.y, p.z).into(),
trans.forward(),
joint.radius,
Color::ORANGE_RED,
);
}
} else {
info!("left_hand_poses returned None");
}
if let Some(joints) = handtracking_ref.get_right_poses() {
for joint in joints {
let p = joint.pose.position;
let r = joint.pose.orientation;
let quat = r.to_quat();
let trans = Transform::from_rotation(quat);
gizmos.circle(
(p.x, p.y, p.z).into(),
trans.forward(),
joint.radius,
Color::LIME_GREEN,
);
}
} else {
info!("right_hand_poses returned None");
}
//lock frame
let frame_state = *frame_state.lock().unwrap();
//get controller

39
src/xr_input/hand.rs
View File

@@ -15,6 +15,7 @@ use crate::{
use super::{
hand_poses::get_simulated_open_hand_transforms,
handtracking::HandTrackingTracker,
oculus_touch::OculusController,
trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker},
Hand,
@@ -319,6 +320,38 @@ pub enum HandBone {
LittleDistal,
LittleTip,
}
impl HandBone {
pub fn get_all_bones() -> [HandBone; 26] {
[
HandBone::Palm,
HandBone::Wrist,
HandBone::ThumbMetacarpal,
HandBone::ThumbProximal,
HandBone::ThumbDistal,
HandBone::ThumbTip,
HandBone::IndexMetacarpal,
HandBone::IndexProximal,
HandBone::IndexIntermediate,
HandBone::IndexDistal,
HandBone::IndexTip,
HandBone::MiddleMetacarpal,
HandBone::MiddleProximal,
HandBone::MiddleIntermediate,
HandBone::MiddleDistal,
HandBone::MiddleTip,
HandBone::RingMetacarpal,
HandBone::RingProximal,
HandBone::RingIntermediate,
HandBone::RingDistal,
HandBone::RingTip,
HandBone::LittleMetacarpal,
HandBone::LittleProximal,
HandBone::LittleIntermediate,
HandBone::LittleDistal,
HandBone::LittleTip,
]
}
}
pub fn update_hand_states(
oculus_controller: Res<OculusController>,
@@ -1015,6 +1048,7 @@ pub fn update_hand_skeletons(
hand_states_option: Option<ResMut<HandStatesResource>>,
mut hand_bone_query: Query<(&mut Transform, &HandBone, &Hand)>,
input_source: Option<Res<HandInputSource>>,
hand_tracking: Res<HandTrackingTracker>,
) {
match input_source {
Some(res) => match *res {
@@ -1048,10 +1082,7 @@ pub fn update_hand_skeletons(
None => info!("hand states resource not initialized yet"),
}
}
HandInputSource::OpenXr => {
info!("hand input source is open XR: this is not implemented yet");
return;
}
HandInputSource::OpenXr => {}
},
None => {
info!("hand input source not initialized");

62
src/xr_input/handtracking.rs Normal file
View File

@@ -0,0 +1,62 @@
use bevy::prelude::*;
use openxr::{HandJointLocationEXT, HandTracker, Result};
use crate::{
input::XrInput,
resources::{XrFrameState, XrFrameWaiter, XrSession},
};
#[derive(Resource)]
pub struct HandTrackingTracker {
left_hand: HandTracker,
right_hand: HandTracker,
}
impl HandTrackingTracker {
pub fn new(session: &XrSession) -> Result<HandTrackingTracker> {
let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
Ok(HandTrackingTracker {
left_hand: left,
right_hand: right,
})
}
pub fn get_ref<'a>(
&'a self,
input: &'a XrInput,
frame_state: &'a XrFrameState,
) -> HandTrackingRef<'a> {
HandTrackingRef {
tracking: self,
input,
frame_state,
}
}
}
pub struct HandTrackingRef<'a> {
tracking: &'a HandTrackingTracker,
input: &'a XrInput,
frame_state: &'a XrFrameState,
}
impl<'a> HandTrackingRef<'a> {
pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
self.input
.stage
.locate_hand_joints(
&self.tracking.left_hand,
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
}
pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT; 26]> {
self.input
.stage
.locate_hand_joints(
&self.tracking.right_hand,
self.frame_state.lock().unwrap().predicted_display_time,
)
.unwrap()
}
}

1
src/xr_input/mod.rs
View File

@@ -7,6 +7,7 @@ pub mod trackers;
pub mod xr_camera;
pub mod hand_poses;
pub mod hand;
pub mod handtracking;
use crate::resources::XrSession;
use crate::xr_begin_frame;