stuff

Cargo.toml

@@ -10,7 +10,7 @@ default = ["openxr/mint", "linked"]
 linked = ["openxr/linked"]
 
 [workspace]
-members = [ "examples/android" ]
+members = ["examples/android", "examples/demo"]
 
 [dependencies]
 anyhow = "1.0.75"
@@ -31,10 +31,7 @@ openxr = { version = "0.17.1", features = ["static"] }
 bevy = "0.12"
 color-eyre = "0.6.2"
 # bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
-bevy_rapier3d = { git = "https://github.com/alexichepura/bevy_rapier", version = "0.22.0", branch = "bevy-012"}
-
-[workspace]
-members = ["examples/demo"]
+bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", version = "0.22.0", branch = "bevy-0.12" }
 
 [[example]]
 name = "xr"

examples/demo/Cargo.toml

@@ -3,22 +3,13 @@ name = "demo"
 version = "0.1.0"
 edition = "2021"
 
-
-[lib]
-crate-type = ["cdylib","lib"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[lib]
+crate-type = ["cdylib", "lib"]
+
 [dependencies]
-bevy = { git = "https://github.com/bevyengine/bevy.git" }
-# bevy = "0.11.3"
-# default-features is false because it for some reason complains when trying to statically link openxr
-bevy_openxr = { path = "../../", default-features = false}
-# bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"}
-bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
+bevy = "0.12"
+bevy_oxr = { path = "../../", default-features = false }
+bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", branch = "bevy-0.12" }
 color-eyre = "0.6.2"
-
-[profile.release]
-lto = "fat"
-codegen-units = 1
-panic = "abort"
-

examples/demo/src/lib.rs

@@ -1,22 +1,25 @@
+use std::{f32::consts::PI, ops::Mul, time::Duration};
+
 use bevy::{
     diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin},
+    ecs::schedule::ScheduleLabel,
     log::info,
     prelude::{
-        *,
-        bevy_main, default, shape, App, Assets, Color, Commands, Component, Event, EventReader,
-        EventWriter, GlobalTransform, IntoSystemConfigs, IntoSystemSetConfigs, Mesh, PbrBundle,
-        PostUpdate, Query, Res, ResMut, Resource, SpatialBundle, StandardMaterial, Startup,
-        Transform, Update, With, Without,
+        default, shape, App, Assets, Color, Commands, Component, Entity, Event, EventReader,
+        EventWriter, FixedUpdate, Gizmos, GlobalTransform, IntoSystemConfigs, IntoSystemSetConfigs,
+        Mesh, PbrBundle, PostUpdate, Quat, Query, Res, ResMut, Resource, Schedule, SpatialBundle,
+        StandardMaterial, Startup, Transform, Update, Vec3, Vec3Swizzles, With, Without, World, bevy_main,
     },
-    time::{Time, Timer},
+    time::{Fixed, Time, Timer},
     transform::TransformSystem,
 };
-use bevy_openxr::{
+use bevy_oxr::{
     input::XrInput,
     resources::{XrFrameState, XrInstance, XrSession},
     xr_input::{
         debug_gizmos::OpenXrDebugRenderer,
-        hand::{HandBone, HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
+        hand::{ HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
+        hands::HandBone,
         interactions::{
             draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
             update_interactable_states, InteractionEvent, Touched, XRDirectInteractor,
@@ -51,7 +54,7 @@ pub fn main() {
         .add_plugins(OpenXrDebugRenderer)
         //rapier goes here
         .add_plugins(RapierPhysicsPlugin::<NoUserData>::default().with_default_system_setup(false))
-        .add_plugins(RapierDebugRenderPlugin::default())
+        // .add_plugins(RapierDebugRenderPlugin::default())
         //lets setup the starting scene
         .add_systems(Startup, setup_scene)
         .add_systems(Startup, spawn_controllers_example) //you need to spawn controllers or it crashes TODO:: Fix this
@@ -89,9 +92,24 @@ pub fn main() {
         .add_plugins(OpenXrHandInput)
         .add_plugins(HandInputDebugRenderer)
         .add_systems(Startup, spawn_physics_hands)
-        .add_systems(Update, update_physics_hands);
+        .add_systems(
+            FixedUpdate,
+            update_physics_hands.before(PhysicsSet::SyncBackend),
+        );
 
     //configure rapier sets
+    let mut physics_schedule = Schedule::new(PhysicsSchedule);
+
+    physics_schedule.configure_sets(
+        (
+            PhysicsSet::SyncBackend,
+            PhysicsSet::StepSimulation,
+            PhysicsSet::Writeback,
+        )
+            .chain()
+            .before(TransformSystem::TransformPropagate),
+    );
+
     app.configure_sets(
         PostUpdate,
         (
@@ -103,24 +121,41 @@ pub fn main() {
             .before(TransformSystem::TransformPropagate),
     );
     //add rapier systems
-    app.add_systems(
-        PostUpdate,
-        (
-            RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::SyncBackend)
-                .in_set(PhysicsSet::SyncBackend),
-            (
-                RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::StepSimulation),
-                // despawn_one_box,
-            )
-                .in_set(PhysicsSet::StepSimulation),
-            RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::Writeback)
-                .in_set(PhysicsSet::Writeback),
-        ),
-    );
-
+    physics_schedule.add_systems((
+        RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::SyncBackend)
+            .in_set(PhysicsSet::SyncBackend),
+        RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::StepSimulation)
+            .in_set(PhysicsSet::StepSimulation),
+        RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::Writeback)
+            .in_set(PhysicsSet::Writeback),
+    ));
+    app.add_schedule(physics_schedule) // configure our fixed timestep schedule to run at the rate we want
+        .insert_resource(Time::<Fixed>::from_duration(Duration::from_secs_f32(
+            FIXED_TIMESTEP,
+        )))
+        .add_systems(FixedUpdate, run_physics_schedule)
+        .add_systems(Startup, configure_physics);
     app.run();
 }
 
+//fixed timesteps?
+const FIXED_TIMESTEP: f32 = 1. / 90.;
+
+// A label for our new Schedule!
+#[derive(ScheduleLabel, Debug, Hash, PartialEq, Eq, Clone)]
+struct PhysicsSchedule;
+
+fn run_physics_schedule(world: &mut World) {
+    world.run_schedule(PhysicsSchedule);
+}
+
+fn configure_physics(mut rapier_config: ResMut<RapierConfiguration>) {
+    rapier_config.timestep_mode = TimestepMode::Fixed {
+        dt: FIXED_TIMESTEP,
+        substeps: 1,
+    }
+}
+
 fn spawn_controllers_example(mut commands: Commands) {
     //left hand
     commands.spawn((
@@ -244,13 +279,21 @@ fn spawn_physics_hands(mut commands: Commands) {
         PhysicsHandBone::LittleDistal,
         PhysicsHandBone::LittleTip,
     ];
-    //lets just do the Right ThumbMetacarpal for now
-    //i dont understand the groups yet
-    let self_group = Group::GROUP_1;
-    let interaction_group = Group::ALL;
     let radius = 0.010;
+    let left_hand_membership_group = Group::GROUP_1;
+    let right_hand_membership_group = Group::GROUP_2;
+    let floor_membership = Group::GROUP_3;
+
 
     for hand in hands.iter() {
+        let hand_membership =  match hand
+         {
+            Hand::Left => left_hand_membership_group,
+            Hand::Right => right_hand_membership_group,
+        };
+        let mut hand_filter: Group = Group::ALL;
+        hand_filter.remove(hand_membership);
+        hand_filter.remove(floor_membership);
         for bone in bones.iter() {
             //spawn the thing
             commands.spawn((
@@ -268,8 +311,9 @@ fn spawn_physics_hands(mut commands: Commands) {
                     },
                     radius,
                 ),
-                RigidBody::KinematicPositionBased,
-                // CollisionGroups::new(self_group, interaction_group),
+                RigidBody::Dynamic,
+                Velocity::default(),
+                CollisionGroups::new(hand_membership, Group::from_bits(0b0001).unwrap()),
                 // SolverGroups::new(self_group, interaction_group),
                 bone.clone(),
                 BoneInitState::False,
@@ -279,6 +323,11 @@ fn spawn_physics_hands(mut commands: Commands) {
     }
 }
 
+pub enum MatchingType {
+    PositionMatching,
+    VelocityMatching,
+}
+
 fn update_physics_hands(
     hands_res: Option<Res<HandsResource>>,
     mut bone_query: Query<(
@@ -287,9 +336,13 @@ fn update_physics_hands(
         &PhysicsHandBone,
         &mut BoneInitState,
         &Hand,
+        &mut Velocity,
     )>,
     hand_query: Query<(&Transform, &HandBone, &Hand, Without<PhysicsHandBone>)>,
+    time: Res<Time>,
+    mut gizmos: Gizmos,
 ) {
+    let matching = MatchingType::VelocityMatching;
     //sanity check do we even have hands?
     match hands_res {
         Some(res) => {
@@ -316,12 +369,45 @@ fn update_physics_hands(
 
                     match *bone.3 {
                         BoneInitState::True => {
-                            //if we are init then we just move em?
-                            *bone.0 = start_components
-                                .unwrap()
-                                .0
-                                .clone()
-                                .looking_at(end_components.unwrap().0.translation, Vec3::Y);
+                            match matching {
+                                MatchingType::PositionMatching => {
+                                    //if we are init then we just move em?
+                                    *bone.0 = start_components
+                                        .unwrap()
+                                        .0
+                                        .clone()
+                                        .looking_at(end_components.unwrap().0.translation, Vec3::Y);
+                                }
+                                MatchingType::VelocityMatching => {
+                                    //calculate position difference
+                                    let diff = (start_components.unwrap().0.translation
+                                        - bone.0.translation)
+                                        / time.delta_seconds();
+                                    bone.5.linvel = diff;
+                                    //calculate angular velocity?
+                                    // gizmos.ray(bone.0.translation, bone.0.forward(), Color::WHITE);
+                                    let desired_forward = start_components
+                                        .unwrap()
+                                        .0
+                                        .clone()
+                                        .looking_at(end_components.unwrap().0.translation, Vec3::Y)
+                                        .rotation;
+                                    // gizmos.ray(
+                                    //     bone.0.translation,
+                                    //     desired_forward.mul_vec3(-Vec3::Z),
+                                    //     Color::GREEN,
+                                    // );
+                                    let cross =
+                                        bone.0.forward().cross(desired_forward.mul_vec3(-Vec3::Z));
+
+                                    // gizmos.ray(
+                                    //     bone.0.translation,
+                                    //     cross,
+                                    //     Color::RED,
+                                    // );
+                                    bone.5.angvel = cross / time.delta_seconds();
+                                }
+                            }
                         }
                         BoneInitState::False => {
                             //build a new collider?
@@ -459,7 +545,7 @@ fn cube_spawner(
     mut materials: ResMut<Assets<StandardMaterial>>,
     mut events: EventReader<SpawnCubeRequest>,
 ) {
-    for request in events.iter() {
+    for request in events.read() {
         // cube
         commands.spawn((
             PbrBundle {

examples/demo/src/setup.rs

@@ -6,9 +6,9 @@ use bevy::{
     transform::TransformBundle,
     utils::default,
 };
-use bevy_openxr::xr_input::interactions::{Touched, XRInteractable, XRInteractableState};
+use bevy_oxr::xr_input::interactions::{Touched, XRInteractable, XRInteractableState};
 use bevy_rapier3d::{
-    prelude::{Collider, RigidBody},
+    prelude::{Collider, RigidBody, Group, CollisionGroups},
     render::ColliderDebugColor,
 };
 
@@ -34,7 +34,9 @@ pub fn setup_scene(
             transform: Transform::from_xyz(0.0, ground_height, 0.0),
             ..default()
         },
+        RigidBody::Fixed,
         Collider::cuboid(ground_size, ground_thickness, ground_size),
+        CollisionGroups::new(Group::GROUP_3, Group::ALL),
     ));
     // cube
     commands.spawn((

src/graphics/mod.rs

@@ -14,6 +14,9 @@ use openxr as xr;
 
 pub fn initialize_xr_graphics(
     window: Option<RawHandleWrapper>,
+    // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+    // any extension at some point
+    enable_hand_tracking: bool,
 ) -> anyhow::Result<(
     RenderDevice,
     RenderQueue,
@@ -31,8 +34,11 @@ pub fn initialize_xr_graphics(
     XrInput,
     XrViews,
     XrFrameState,
+    // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+    // any extension at some point
+    bool,
 )> {
-    vulkan::initialize_xr_graphics(window)
+    vulkan::initialize_xr_graphics(window,enable_hand_tracking)
 }
 
 pub fn xr_entry() -> xr::Entry {

src/graphics/vulkan.rs

@@ -20,6 +20,9 @@ use crate::VIEW_TYPE;
 
 pub fn initialize_xr_graphics(
     window: Option<RawHandleWrapper>,
+    // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+    // any extension at some point
+    enable_hand_tracking: bool,
 ) -> anyhow::Result<(
     RenderDevice,
     RenderQueue,
@@ -37,6 +40,9 @@ pub fn initialize_xr_graphics(
     XrInput,
     XrViews,
     XrFrameState,
+    // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+    // any extension at some point
+    bool,
 )> {
     use wgpu_hal::{api::Vulkan as V, Api};
 
@@ -51,16 +57,16 @@ pub fn initialize_xr_graphics(
 
     let mut enabled_extensions = xr::ExtensionSet::default();
     enabled_extensions.khr_vulkan_enable2 = true;
-    enabled_extensions.khr_convert_timespec_time = true;
-    enabled_extensions.other.push("XR_METAX2_detached_controllers".into());
     #[cfg(target_os = "android")]
     {
         enabled_extensions.khr_android_create_instance = true;
     }
-    enabled_extensions.ext_hand_tracking = available_extensions.ext_hand_tracking;
+    // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+    // any extension at some point
+    enabled_extensions.ext_hand_tracking =
+        available_extensions.ext_hand_tracking && enable_hand_tracking;
     // enabled_extensions.ext_hand_joints_motion_range = available_extensions.ext_hand_joints_motion_range;
 
-
     let available_layers = xr_entry.enumerate_layers()?;
     info!("available xr layers: {:#?}", available_layers);
 
@@ -407,6 +413,9 @@ pub fn initialize_xr_graphics(
             should_render: true,
         })
         .into(),
+        // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+        // any extension at some point
+        available_extensions.ext_hand_tracking && enable_hand_tracking,
     ))
 }
 

src/lib.rs

@@ -7,6 +7,7 @@ pub mod xr_input;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
+use crate::xr_input::hands::hand_tracking::DisableHandTracking;
 use crate::xr_input::oculus_touch::ActionSets;
 use bevy::app::PluginGroupBuilder;
 use bevy::ecs::system::SystemState;
@@ -21,6 +22,8 @@ use input::XrInput;
 use openxr as xr;
 use resources::*;
 use xr_input::controllers::XrControllerType;
+use xr_input::hands::emulated::EmulatedHandsPlugin;
+use xr_input::hands::hand_tracking::HandTrackingPlugin;
 use xr_input::handtracking::HandTrackingTracker;
 use xr_input::OpenXrInput;
 
@@ -30,8 +33,13 @@ pub const LEFT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHan
 pub const RIGHT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(3383858418);
 
 /// Adds OpenXR support to an App
-#[derive(Default)]
-pub struct OpenXrPlugin;
+pub struct OpenXrPlugin(pub bool);
+
+impl Default for OpenXrPlugin {
+    fn default() -> Self {
+        OpenXrPlugin(true)
+    }
+}
 
 #[derive(Resource)]
 pub struct FutureXrResources(
@@ -80,7 +88,8 @@ impl Plugin for OpenXrPlugin {
             input,
             views,
             frame_state,
-        ) = graphics::initialize_xr_graphics(primary_window).unwrap();
+            hand_tracking_enabled,
+        ) = graphics::initialize_xr_graphics(primary_window,self.0).unwrap();
         // std::thread::sleep(Duration::from_secs(5));
         debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
         debug!("Configured wgpu adapter Features: {:#?}", device.features());
@@ -98,6 +107,9 @@ impl Plugin for OpenXrPlugin {
             views,
             frame_state,
         ));
+        if !hand_tracking_enabled {
+            app.insert_resource(DisableHandTracking::Both);
+        }
         app.insert_resource(ActionSets(vec![]));
         app.add_plugins(RenderPlugin {
             render_creation: RenderCreation::Manual(
@@ -202,8 +214,9 @@ impl PluginGroup for DefaultXrPlugins {
             .build()
             .disable::<RenderPlugin>()
             .disable::<PipelinedRenderingPlugin>()
-            .add_before::<RenderPlugin, _>(OpenXrPlugin)
+            .add_before::<RenderPlugin, _>(OpenXrPlugin::default())
             .add_after::<OpenXrPlugin, _>(OpenXrInput::new(XrControllerType::OculusTouch))
+            .add(EmulatedHandsPlugin).add(HandTrackingPlugin)
             .set(WindowPlugin {
                 #[cfg(not(target_os = "android"))]
                 primary_window: Some(Window {

src/xr_input/hand.rs

@@ -18,7 +18,7 @@ use super::{
     handtracking::HandTrackingTracker,
     oculus_touch::OculusController,
     trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTracker, OpenXRTrackingRoot},
-    Hand, QuatConv, hands::HandBone,
+    Hand, QuatConv, hands::{HandBone, BoneTrackingStatus},
 };
 
 /// add debug renderer for controllers
@@ -27,11 +27,12 @@ pub struct OpenXrHandInput;
 
 impl Plugin for OpenXrHandInput {
     fn build(&self, app: &mut bevy::prelude::App) {
-        app.add_systems(Update, update_hand_skeletons)
-            .add_systems(PreUpdate, update_hand_states)
-            .add_systems(Startup, spawn_hand_entities)
-            .insert_resource(HandStatesResource::default())
-            .insert_resource(HandInputSource::default());
+        app
+            // .add_systems(Update, update_hand_skeletons)
+            // .add_systems(PreUpdate, update_hand_states)
+            .add_systems(Startup, spawn_hand_entities);
+            // .insert_resource(HandStatesResource::default())
+            // .insert_resource(HandInputSource::default());
     }
 }
 
@@ -195,6 +196,7 @@ pub fn spawn_hand_entities(mut commands: Commands) {
                     bone.clone(),
                     OpenXRTracker,
                     hand.clone(),
+                    BoneTrackingStatus::Emulated,
                 ))
                 .id();
             match hand {

src/xr_input/hands/emulated.rs

@@ -1,18 +1,540 @@
-use bevy::prelude::*;
+use std::f32::consts::PI;
 
-use crate::xr_input::Hand;
+use bevy::prelude::*;
+use openxr::{Action, ActionTy, Binding, HandJoint};
+
+use crate::{
+    resources::{XrInstance, XrSession},
+    xr_input::{
+        controllers::Touchable,
+        hand_poses::get_simulated_open_hand_transforms,
+        oculus_touch::ActionSets,
+        trackers::{OpenXRLeftController, OpenXRRightController},
+        Hand,
+    },
+};
 
 use super::HandBone;
 
-#[derive(Deref, DerefMut, Resource)]
-pub struct EmulatedHandPose(pub Box<dyn Fn(Hand, HandBone) -> (Vec3, Quat) + Send + Sync>);
+pub enum TouchValue<T: ActionTy> {
+    None,
+    Touched(T),
+}
+
+// #[derive(Deref, DerefMut, Resource)]
+// pub struct EmulatedHandPoseFunctions {
+//     pub get_base_pose: Box<dyn Fn(Hand) -> [Transform; 26] + Send + Sync>,
+//     pub map_data: Box<dyn Fn(Hand) -> [Transform; 26] + Send + Sync>,
+// }
 
 pub struct EmulatedHandsPlugin;
 
 impl Plugin for EmulatedHandsPlugin {
     fn build(&self, app: &mut App) {
-        app.add_systems(Update, update_hand_skeleton_from_emulated);
+        app.add_systems(PreUpdate, update_hand_skeleton_from_emulated);
+        app.add_systems(
+            Startup,
+            setup_hand_emulation_action_set.map(|res| res.unwrap()),
+        );
     }
 }
+#[derive(Resource)]
+pub struct HandEmulationActionSet {
+    thumb_touch: Action<bool>,
+    thumb_x: Action<f32>,
+    thumb_y: Action<f32>,
+    index: Touchable<f32>,
+    middle: Touchable<f32>,
+    ring: Touchable<f32>,
+    little: Touchable<f32>,
+}
 
-pub(crate) fn update_hand_skeleton_from_emulated() {}
+fn setup_hand_emulation_action_set(
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+    mut action_sets: ResMut<ActionSets>,
+    mut commands: Commands,
+) -> anyhow::Result<()> {
+    let left_path = instance.string_to_path("/user/hand/left").unwrap();
+    let right_path = instance.string_to_path("/user/hand/right").unwrap();
+    let hands = [left_path, right_path];
+    // This unwrap Should not trigger since both strings are not empty
+    let action_set = instance
+        .create_action_set("hand_pose_approximation_set", "Hand Pose Approximaiton", 0)
+        .unwrap();
+    let hand_action_set = HandEmulationActionSet {
+        thumb_touch: action_set.create_action::<bool>("thumb_touch", "Thumb Touched", &hands)?,
+        thumb_x: action_set.create_action::<f32>("thumb_x", "Thumb X", &hands)?,
+        thumb_y: action_set.create_action::<f32>("thumb_y", "Thumb Y", &hands)?,
+
+        index: Touchable::<f32> {
+            inner: action_set.create_action("index_value", "Index Finger Pull", &hands)?,
+            touch: action_set.create_action("index_touch", "Index Finger Touch", &hands)?,
+        },
+        middle: Touchable::<f32> {
+            inner: action_set.create_action("middle_value", "Middle Finger Pull", &hands)?,
+            touch: action_set.create_action("middle_touch", "Middle Finger Touch", &hands)?,
+        },
+        ring: Touchable::<f32> {
+            inner: action_set.create_action("ring_value", "Ring Finger Pull", &hands)?,
+            touch: action_set.create_action("ring_touch", "Ring Finger Touch", &hands)?,
+        },
+        little: Touchable::<f32> {
+            inner: action_set.create_action("little_value", "Little Finger Pull", &hands)?,
+            touch: action_set.create_action("little_touch", "Little Finger Touch", &hands)?,
+        },
+    };
+
+    suggest_oculus_touch_profile(&instance, &hand_action_set)?;
+
+    session.attach_action_sets(&[&action_set])?;
+
+    action_sets.0.push(action_set);
+
+    commands.insert_resource(hand_action_set);
+
+    Ok(())
+}
+
+pub struct EmulatedHandPoseData {}
+
+fn bind<'a, T: ActionTy>(
+    action: &'a Action<T>,
+    path: &str,
+    i: &XrInstance,
+    bindings: &mut Vec<Binding<'a>>,
+) -> anyhow::Result<()> {
+    bindings.push(Binding::new(
+        &action,
+        i.string_to_path(&("/user/hand/left/input".to_string() + path))?,
+    ));
+    bindings.push(Binding::new(
+        &action,
+        i.string_to_path(&("/user/hand/right/input".to_string() + path))?,
+    ));
+    Ok(())
+}
+fn bind_single<'a, T: ActionTy>(
+    action: &'a Action<T>,
+    path: &str,
+    hand: Hand,
+    i: &XrInstance,
+    bindings: &mut Vec<Binding<'a>>,
+) -> anyhow::Result<()> {
+    match hand {
+        Hand::Left => bindings.push(Binding::new(
+            &action,
+            i.string_to_path(&("/user/hand/left/input".to_string() + path))?,
+        )),
+        Hand::Right => bindings.push(Binding::new(
+            &action,
+            i.string_to_path(&("/user/hand/right/input".to_string() + path))?,
+        )),
+    }
+    Ok(())
+}
+
+fn suggest_oculus_touch_profile(
+    i: &XrInstance,
+    action_set: &HandEmulationActionSet,
+) -> anyhow::Result<()> {
+    let mut b: Vec<Binding> = Vec::new();
+    bind(&action_set.thumb_x, "/thumbstick/x", i, &mut b)?;
+    bind(&action_set.thumb_y, "/thumbstick/y", i, &mut b)?;
+    bind(&action_set.thumb_touch, "/thumbstick/touch", i, &mut b)?;
+    bind(&action_set.thumb_touch, "/thumbrest/touch", i, &mut b)?;
+    // bind_single(&action_set.thumb_touch, "/x/touch", Hand::Left, i, &mut b)?;
+    // bind_single(&action_set.thumb_touch, "/y/touch", Hand::Left, i, &mut b)?;
+    // bind_single(&action_set.thumb_touch, "/a/touch", Hand::Right, i, &mut b)?;
+    // bind_single(&action_set.thumb_touch, "/b/touch", Hand::Right, i, &mut b)?;
+    
+    // bind(&action_set.index.touch, "/trigger/touch", i, &mut b)?;
+    // bind(&action_set.index.inner, "/trigger/value", i, &mut b)?;
+    //
+    // bind(&action_set.middle.touch, "/squeeze/touch", i, &mut b)?;
+    // bind(&action_set.middle.inner, "/squeeze/value", i, &mut b)?;
+    // bind(&action_set.ring.touch, "/squeeze/touch", i, &mut b)?;
+    // bind(&action_set.ring.inner, "/squeeze/value", i, &mut b)?;
+    // bind(&action_set.little.touch, "/squeeze/touch", i, &mut b)?;
+    // bind(&action_set.little.inner, "/squeeze/value", i, &mut b)?;
+
+    i.suggest_interaction_profile_bindings(
+        i.string_to_path("/interaction_profiles/oculus/touch_controller")?,
+        &b,
+    )?;
+    Ok(())
+}
+
+pub(crate) fn update_hand_skeleton_from_emulated(
+    session: Res<XrSession>,
+    instance: Res<XrInstance>,
+    action_set: Res<HandEmulationActionSet>,
+    left_controller_transform: Query<&Transform, With<OpenXRLeftController>>,
+    right_controller_transform: Query<&Transform, With<OpenXRRightController>>,
+    mut bones: Query<
+        (&mut Transform, &HandBone, &Hand),
+        (
+            Without<OpenXRLeftController>,
+            Without<OpenXRRightController>,
+        ),
+    >,
+) {
+    let mut data: [[Transform; 26]; 2] = [[Transform::default(); 26]; 2];
+    for (subaction_path, hand) in [
+        (
+            instance.string_to_path("/user/hand/left").unwrap(),
+            Hand::Left,
+        ),
+        (
+            instance.string_to_path("/user/hand/right").unwrap(),
+            Hand::Right,
+        ),
+    ] {
+        let thumb_curl = match action_set
+            .thumb_touch
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state
+        {
+            true => 1.0,
+            false => 0.0,
+        };
+        let index_curl = action_set
+            .index
+            .inner
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        let middle_curl = action_set
+            .middle
+            .inner
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        let ring_curl = action_set
+            .ring
+            .inner
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        let little_curl = action_set
+            .little
+            .inner
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        data[match hand {
+            Hand::Left => 0,
+            Hand::Right => 1,
+        }] = update_hand_bones_emulated(
+            match hand {
+                Hand::Left => left_controller_transform.single(),
+                Hand::Right => right_controller_transform.single(),
+            },
+            hand,
+            thumb_curl,
+            index_curl,
+            middle_curl,
+            ring_curl,
+            little_curl,
+        );
+    }
+    for (mut t, bone, hand) in bones.iter_mut() {
+        *t = data[match hand {
+            Hand::Left => 0,
+            Hand::Right => 1,
+        }][bone.get_index_from_bone()]
+    }
+}
+pub fn update_hand_bones_emulated(
+    controller_transform: &Transform,
+    hand: Hand,
+    thumb_curl: f32,
+    index_curl: f32,
+    middle_curl: f32,
+    ring_curl: f32,
+    little_curl: f32,
+) -> [Transform; 26] {
+    let left_hand_rot = Quat::from_rotation_y(PI);
+    let hand_translation: Vec3 = controller_transform.translation;
+
+    let controller_quat: Quat = match hand {
+        Hand::Left => controller_transform.rotation.mul_quat(left_hand_rot),
+        Hand::Right => controller_transform.rotation,
+    };
+
+    let splay_direction = match hand {
+        Hand::Left => -1.0,
+        Hand::Right => 1.0,
+    };
+    //lets make a structure to hold our calculated transforms for now
+    let mut calc_transforms = [Transform::default(); 26];
+
+    //get palm quat
+    let y = Quat::from_rotation_y(-90.0 * PI / 180.0);
+    let x = Quat::from_rotation_x(-90.0 * PI / 180.0);
+    let palm_quat = controller_quat.mul_quat(y).mul_quat(x);
+    //get simulated bones
+    let hand_transform_array: [Transform; 26] = get_simulated_open_hand_transforms(hand);
+    //palm
+    let palm = hand_transform_array[HandJoint::PALM];
+    calc_transforms[HandJoint::PALM] = Transform {
+        translation: hand_translation + palm.translation,
+        ..default()
+    };
+    //wrist
+    let wrist = hand_transform_array[HandJoint::WRIST];
+    calc_transforms[HandJoint::WRIST] = Transform {
+        translation: hand_translation + palm.translation + palm_quat.mul_vec3(wrist.translation),
+        ..default()
+    };
+
+    //thumb
+    let thumb_joints = [
+        HandJoint::THUMB_METACARPAL,
+        HandJoint::THUMB_PROXIMAL,
+        HandJoint::THUMB_DISTAL,
+        HandJoint::THUMB_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * 30.0 * PI / 180.0);
+    let huh = Quat::from_rotation_x(-35.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(huh).mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, thumb_curl);
+                let tp_lrot = Quat::from_rotation_y(splay_direction * curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+
+    //index
+    let thumb_joints = [
+        HandJoint::INDEX_METACARPAL,
+        HandJoint::INDEX_PROXIMAL,
+        HandJoint::INDEX_INTERMEDIATE,
+        HandJoint::INDEX_DISTAL,
+        HandJoint::INDEX_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * 10.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, index_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+
+    //middle
+    let thumb_joints = [
+        HandJoint::MIDDLE_METACARPAL,
+        HandJoint::MIDDLE_PROXIMAL,
+        HandJoint::MIDDLE_INTERMEDIATE,
+        HandJoint::MIDDLE_DISTAL,
+        HandJoint::MIDDLE_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * 0.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, middle_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+    //ring
+    let thumb_joints = [
+        HandJoint::RING_METACARPAL,
+        HandJoint::RING_PROXIMAL,
+        HandJoint::RING_INTERMEDIATE,
+        HandJoint::RING_DISTAL,
+        HandJoint::RING_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * -10.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, ring_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+
+    //little
+    let thumb_joints = [
+        HandJoint::LITTLE_METACARPAL,
+        HandJoint::LITTLE_PROXIMAL,
+        HandJoint::LITTLE_INTERMEDIATE,
+        HandJoint::LITTLE_DISTAL,
+        HandJoint::LITTLE_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * -20.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, little_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+    calc_transforms
+}
+
+fn get_bone_curl_angle(bone: HandJoint, thumb_curl: f32) -> f32 {
+    todo!()
+}

src/xr_input/hands/hand_tracking.rs

@@ -117,7 +117,7 @@ impl<'a> HandTrackingRef<'a> {
 impl Plugin for HandTrackingPlugin {
     fn build(&self, app: &mut App) {
         app.add_systems(
-            Update,
+            PreUpdate,
             (update_hand_bones).run_if(|dh: Option<Res<DisableHandTracking>>| {
                 !dh.is_some_and(|v| *v == DisableHandTracking::Both)
             }),
@@ -126,6 +126,7 @@ impl Plugin for HandTrackingPlugin {
 }
 
 pub fn update_hand_bones(
+    disabled_tracking: Res<DisableHandTracking>,
     hand_tracking: Res<HandTrackingData>,
     xr_input: Res<XrInput>,
     xr_frame_state: Res<XrFrameState>,
@@ -145,6 +146,17 @@ pub fn update_hand_bones(
     bones
         .par_iter_mut()
         .for_each(|(mut transform, hand, bone, mut radius, mut status)| {
+            match (&hand, disabled_tracking.as_ref()) {
+                (Hand::Left, DisableHandTracking::OnlyLeft) => {
+                    *status = BoneTrackingStatus::Emulated;
+                    return;
+                }
+                (Hand::Right, DisableHandTracking::OnlyRight) => {
+                    *status = BoneTrackingStatus::Emulated;
+                    return;
+                }
+                _ => {}
+            }
             let bone_data = match (hand, &left_hand_data, &right_hand_data) {
                 (Hand::Left, Some(data), _) => data.get_joint(*bone),
                 (Hand::Right, _, Some(data)) => data.get_joint(*bone),

src/xr_input/hands/mod.rs

@@ -45,6 +45,23 @@ pub enum HandBone {
     LittleTip,
 }
 impl HandBone {
+    pub fn is_finger(&self) -> bool {
+        match &self {
+            HandBone::Wrist => false,
+            HandBone::Palm => false,
+            _ => true,
+        }
+    }
+    pub fn is_metacarpal(&self) -> bool {
+        match &self {
+            HandBone::ThumbMetacarpal => true,
+            HandBone::IndexMetacarpal => true,
+            HandBone::MiddleMetacarpal => true,
+            HandBone::RingMetacarpal => true,
+            HandBone::LittleTip => true,
+            _ => false,
+        }
+    }
     pub const fn get_all_bones() -> [HandBone; 26] {
         [
             HandBone::Palm,