Merge pull request #41 from Schmarni-Dev/hand_refactor

Hand refactor

.gitignore

@@ -1,3 +1,4 @@
-/target
+**/target
-/Cargo.lock
+**/Cargo.lock
+**/runtime_libs/arm64-v8a/*
 \.DS_Store

examples/demo/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [lib]
-crate-type = ["lib", "cdylib"]
+crate-type = ["rlib", "cdylib"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

examples/demo/Cargo_back.toml

@@ -0,0 +1,24 @@
+[package]
+name = "demo"
+version = "0.1.0"
+edition = "2021"
+
+
+[lib]
+crate-type = ["cdylib", "lib"]
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bevy = { git = "https://github.com/bevyengine/bevy.git" }
+# bevy = "0.11.3"
+# default-features is false because it for some reason complains when trying to statically link openxr
+bevy_oxr = { path = "../../" }
+# bevy_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", default-features = false, branch = "demo"}
+bevy_rapier3d = { git = "https://github.com/devil-ira/bevy_rapier", version = "0.22.0", branch = "bevy-0.12" }
+# bevy_rapier3d = { git = "https://github.com/Schmarni-Dev/bevy_rapier" }
+color-eyre = "0.6.2"
+
+[profile.release]
+lto = "fat"
+codegen-units = 1
+panic = "abort"

examples/demo/Readme.md

@@ -0,0 +1,11 @@
+## setup
+install xbuild ```cargo install --git https://github.com/rust-mobile/xbuild```
+run ```x doctor``` and install all required depencencies
+download the [openxr loader](https://developer.oculus.com/downloads/package/oculus-openxr-mobile-sdk/) and put it in the runtime_libs/arm64-v8a folder
+
+## how to run
+run ```x devices```
+
+and get the device name that looks something like this ```adb:1WD523S``` (probably a bit longer)
+ 
+then ```run x run --release```

examples/demo/src/lib.rs

@@ -18,9 +18,9 @@ use bevy_oxr::{
     input::XrInput,
     resources::{XrFrameState, XrInstance, XrSession},
     xr_input::{
-        actions::XrActionSets,
         debug_gizmos::OpenXrDebugRenderer,
-        hand::{HandBone, HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
+        hands::common::{ HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
+        hands::HandBone,
         interactions::{
             draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
             update_interactable_states, InteractionEvent, Touched, XRDirectInteractor,
@@ -29,7 +29,7 @@ use bevy_oxr::{
         oculus_touch::OculusController,
         prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig},
         trackers::{OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker},
-        Hand,
+        Hand, actions::XrActionSets,
     },
     DefaultXrPlugins,
 };

examples/demo/src/lib_bak.rs

@@ -0,0 +1,609 @@
+use bevy::{
+    diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin},
+    log::info,
+    prelude::{
+        *,
+        bevy_main, default, shape, App, Assets, Color, Commands, Component, Event, EventReader,
+        EventWriter, GlobalTransform, IntoSystemConfigs, IntoSystemSetConfigs, Mesh, PbrBundle,
+        PostUpdate, Query, Res, ResMut, Resource, SpatialBundle, StandardMaterial, Startup,
+        Transform, Update, With, Without,
+    },
+    time::{Time, Timer},
+    transform::TransformSystem,
+};
+use bevy_openxr::{
+    input::XrInput,
+    resources::{XrFrameState, XrInstance, XrSession},
+    xr_input::{
+        debug_gizmos::OpenXrDebugRenderer,
+        hand::{HandBone, HandInputDebugRenderer, HandResource, HandsResource, OpenXrHandInput},
+        interactions::{
+            draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
+            update_interactable_states, InteractionEvent, Touched, XRDirectInteractor,
+            XRInteractable, XRInteractableState, XRInteractorState, XRSelection,
+        },
+        oculus_touch::OculusController,
+        prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig},
+        trackers::{OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker},
+        Hand,
+    },
+    DefaultXrPlugins,
+};
+
+mod setup;
+use crate::setup::setup_scene;
+use bevy_rapier3d::prelude::*;
+
+#[bevy_main]
+pub fn main() {
+    color_eyre::install().unwrap();
+
+    info!("Running bevy_openxr demo");
+    let mut app = App::new();
+
+    app
+        //lets get the usual diagnostic stuff added
+        .add_plugins(LogDiagnosticsPlugin::default())
+        .add_plugins(FrameTimeDiagnosticsPlugin)
+        //lets get the xr defaults added
+        .add_plugins(DefaultXrPlugins)
+        //lets add the debug renderer for the controllers
+        .add_plugins(OpenXrDebugRenderer)
+        //rapier goes here
+        .add_plugins(RapierPhysicsPlugin::<NoUserData>::default().with_default_system_setup(false))
+        .add_plugins(RapierDebugRenderPlugin::default())
+        //lets setup the starting scene
+        .add_systems(Startup, setup_scene)
+        .add_systems(Startup, spawn_controllers_example) //you need to spawn controllers or it crashes TODO:: Fix this
+        //add locomotion
+        .add_systems(Update, proto_locomotion)
+        .insert_resource(PrototypeLocomotionConfig::default())
+        //lets add the interaction systems
+        .add_event::<InteractionEvent>()
+        .add_systems(Update, prototype_interaction_input)
+        .add_systems(Update, interactions.before(update_interactable_states))
+        .add_systems(Update, update_interactable_states)
+        .add_systems(
+            Update,
+            socket_interactions.before(update_interactable_states),
+        )
+        //add the grabbable system
+        .add_systems(Update, update_grabbables.after(update_interactable_states))
+        //draw the interaction gizmos
+        .add_systems(
+            Update,
+            draw_interaction_gizmos.after(update_interactable_states),
+        )
+        .add_systems(Update, draw_socket_gizmos.after(update_interactable_states))
+        //add our cube spawning system
+        .add_event::<SpawnCubeRequest>()
+        .insert_resource(SpawnCubeTimer(Timer::from_seconds(
+            0.25,
+            bevy::time::TimerMode::Once,
+        )))
+        .add_systems(Update, request_cube_spawn)
+        .add_systems(Update, cube_spawner.after(request_cube_spawn))
+        //test capsule
+        .add_systems(Startup, spawn_capsule)
+        //physics hands
+        .add_plugins(OpenXrHandInput)
+        .add_plugins(HandInputDebugRenderer)
+        .add_systems(Startup, spawn_physics_hands)
+        .add_systems(Update, update_physics_hands);
+
+    //configure rapier sets
+    app.configure_sets(
+        PostUpdate,
+        (
+            PhysicsSet::SyncBackend,
+            PhysicsSet::StepSimulation,
+            PhysicsSet::Writeback,
+        )
+            .chain()
+            .before(TransformSystem::TransformPropagate),
+    );
+    //add rapier systems
+    app.add_systems(
+        PostUpdate,
+        (
+            RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::SyncBackend)
+                .in_set(PhysicsSet::SyncBackend),
+            (
+                RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::StepSimulation),
+                // despawn_one_box,
+            )
+                .in_set(PhysicsSet::StepSimulation),
+            RapierPhysicsPlugin::<NoUserData>::get_systems(PhysicsSet::Writeback)
+                .in_set(PhysicsSet::Writeback),
+        ),
+    );
+
+    app.run();
+}
+
+fn spawn_controllers_example(mut commands: Commands) {
+    //left hand
+    commands.spawn((
+        OpenXRLeftController,
+        OpenXRController,
+        OpenXRTracker,
+        SpatialBundle::default(),
+        XRDirectInteractor,
+        XRInteractorState::default(),
+        XRSelection::default(),
+    ));
+    //right hand
+    commands.spawn((
+        OpenXRRightController,
+        OpenXRController,
+        OpenXRTracker,
+        SpatialBundle::default(),
+        XRDirectInteractor,
+        XRInteractorState::default(),
+        XRSelection::default(),
+    ));
+}
+
+fn spawn_capsule(
+    mut commands: Commands,
+    mut meshes: ResMut<Assets<Mesh>>,
+    mut materials: ResMut<Assets<StandardMaterial>>,
+) {
+    commands.spawn((
+        PbrBundle {
+            mesh: meshes.add(Mesh::from(shape::Capsule {
+                radius: 0.033,
+                depth: 0.115,
+                ..default()
+            })),
+            material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+            transform: Transform::from_xyz(0.0, 2.0, 0.0),
+            ..default()
+        },
+        // Collider::capsule_y(0.0575, 0.034),
+        Collider::capsule(
+            Vec3 {
+                x: 0.0,
+                y: -0.0575,
+                z: 0.0,
+            },
+            Vec3 {
+                x: 0.0,
+                y: 0.0575,
+                z: 0.0,
+            },
+            0.034,
+        ),
+        RigidBody::Dynamic,
+    ));
+}
+
+#[derive(Component, PartialEq, Debug, Clone, Copy)]
+pub enum PhysicsHandBone {
+    Palm,
+    Wrist,
+    ThumbMetacarpal,
+    ThumbProximal,
+    ThumbDistal,
+    ThumbTip,
+    IndexMetacarpal,
+    IndexProximal,
+    IndexIntermediate,
+    IndexDistal,
+    IndexTip,
+    MiddleMetacarpal,
+    MiddleProximal,
+    MiddleIntermediate,
+    MiddleDistal,
+    MiddleTip,
+    RingMetacarpal,
+    RingProximal,
+    RingIntermediate,
+    RingDistal,
+    RingTip,
+    LittleMetacarpal,
+    LittleProximal,
+    LittleIntermediate,
+    LittleDistal,
+    LittleTip,
+}
+#[derive(Component, PartialEq)]
+pub enum BoneInitState {
+    True,
+    False,
+}
+
+fn spawn_physics_hands(mut commands: Commands) {
+    //here we go
+    let hands = [Hand::Left, Hand::Right];
+    let bones = [
+        PhysicsHandBone::Palm,
+        PhysicsHandBone::Wrist,
+        PhysicsHandBone::ThumbMetacarpal,
+        PhysicsHandBone::ThumbProximal,
+        PhysicsHandBone::ThumbDistal,
+        PhysicsHandBone::ThumbTip,
+        PhysicsHandBone::IndexMetacarpal,
+        PhysicsHandBone::IndexProximal,
+        PhysicsHandBone::IndexIntermediate,
+        PhysicsHandBone::IndexDistal,
+        PhysicsHandBone::IndexTip,
+        PhysicsHandBone::MiddleMetacarpal,
+        PhysicsHandBone::MiddleProximal,
+        PhysicsHandBone::MiddleIntermediate,
+        PhysicsHandBone::MiddleDistal,
+        PhysicsHandBone::MiddleTip,
+        PhysicsHandBone::RingMetacarpal,
+        PhysicsHandBone::RingProximal,
+        PhysicsHandBone::RingIntermediate,
+        PhysicsHandBone::RingDistal,
+        PhysicsHandBone::RingTip,
+        PhysicsHandBone::LittleMetacarpal,
+        PhysicsHandBone::LittleProximal,
+        PhysicsHandBone::LittleIntermediate,
+        PhysicsHandBone::LittleDistal,
+        PhysicsHandBone::LittleTip,
+    ];
+    //lets just do the Right ThumbMetacarpal for now
+    //i dont understand the groups yet
+    let self_group = Group::GROUP_1;
+    let interaction_group = Group::ALL;
+    let radius = 0.010;
+
+    for hand in hands.iter() {
+        for bone in bones.iter() {
+            //spawn the thing
+            commands.spawn((
+                SpatialBundle::default(),
+                Collider::capsule(
+                    Vec3 {
+                        x: 0.0,
+                        y: -0.0575,
+                        z: 0.0,
+                    },
+                    Vec3 {
+                        x: 0.0,
+                        y: 0.0575,
+                        z: 0.0,
+                    },
+                    radius,
+                ),
+                RigidBody::KinematicPositionBased,
+                // CollisionGroups::new(self_group, interaction_group),
+                // SolverGroups::new(self_group, interaction_group),
+                bone.clone(),
+                BoneInitState::False,
+                hand.clone(),
+            ));
+        }
+    }
+}
+
+fn update_physics_hands(
+    hands_res: Option<Res<HandsResource>>,
+    mut bone_query: Query<(
+        &mut Transform,
+        &mut Collider,
+        &PhysicsHandBone,
+        &mut BoneInitState,
+        &Hand,
+    )>,
+    hand_query: Query<(&Transform, &HandBone, &Hand, Without<PhysicsHandBone>)>,
+) {
+    //sanity check do we even have hands?
+    match hands_res {
+        Some(res) => {
+            //config stuff
+            let radius = 0.010;
+            for mut bone in bone_query.iter_mut() {
+                let hand_res = match bone.4 {
+                    Hand::Left => res.left,
+                    Hand::Right => res.right,
+                };
+
+                //lets just do the Right ThumbMetacarpal for now
+                let result = get_start_and_end_entities(hand_res, bone.2);
+                if let Some((start_entity, end_entity)) = result {
+                    //now we need their transforms
+                    let start_components = hand_query.get(start_entity);
+                    let end_components = hand_query.get(end_entity);
+                    let direction = end_components.unwrap().0.translation
+                        - start_components.unwrap().0.translation;
+                    if direction.length() < 0.001 {
+                        //i hate this but we need to skip init if the length is zero
+                        return;
+                    }
+
+                    match *bone.3 {
+                        BoneInitState::True => {
+                            //if we are init then we just move em?
+                            *bone.0 = start_components
+                                .unwrap()
+                                .0
+                                .clone()
+                                .looking_at(end_components.unwrap().0.translation, Vec3::Y);
+                        }
+                        BoneInitState::False => {
+                            //build a new collider?
+                            *bone.1 = Collider::capsule(
+                                Vec3::splat(0.0),
+                                Vec3 {
+                                    x: 0.0,
+                                    y: 0.0,
+                                    z: -direction.length(),
+                                },
+                                radius,
+                            );
+                            *bone.3 = BoneInitState::True;
+                        }
+                    }
+                }
+            }
+        }
+        None => info!("hand states resource not initialized yet"),
+    }
+}
+
+fn get_start_and_end_entities(
+    hand_res: HandResource,
+    bone: &PhysicsHandBone,
+) -> Option<(Entity, Entity)> {
+    match bone {
+        PhysicsHandBone::Palm => return None,
+        PhysicsHandBone::Wrist => return None,
+        PhysicsHandBone::ThumbMetacarpal => {
+            return Some((hand_res.thumb.metacarpal, hand_res.thumb.proximal))
+        }
+        PhysicsHandBone::ThumbProximal => {
+            return Some((hand_res.thumb.proximal, hand_res.thumb.distal))
+        }
+        PhysicsHandBone::ThumbDistal => return Some((hand_res.thumb.distal, hand_res.thumb.tip)),
+        PhysicsHandBone::ThumbTip => return None,
+        PhysicsHandBone::IndexMetacarpal => {
+            return Some((hand_res.index.metacarpal, hand_res.index.proximal))
+        }
+        PhysicsHandBone::IndexProximal => {
+            return Some((hand_res.index.proximal, hand_res.index.intermediate))
+        }
+        PhysicsHandBone::IndexIntermediate => {
+            return Some((hand_res.index.intermediate, hand_res.index.distal))
+        }
+        PhysicsHandBone::IndexDistal => return Some((hand_res.index.distal, hand_res.index.tip)),
+        PhysicsHandBone::IndexTip => return None,
+        PhysicsHandBone::MiddleMetacarpal => {
+            return Some((hand_res.middle.metacarpal, hand_res.middle.proximal))
+        }
+        PhysicsHandBone::MiddleProximal => {
+            return Some((hand_res.middle.proximal, hand_res.middle.intermediate))
+        }
+        PhysicsHandBone::MiddleIntermediate => {
+            return Some((hand_res.middle.intermediate, hand_res.middle.distal))
+        }
+        PhysicsHandBone::MiddleDistal => {
+            return Some((hand_res.middle.distal, hand_res.middle.tip))
+        }
+        PhysicsHandBone::MiddleTip => return None,
+        PhysicsHandBone::RingMetacarpal => {
+            return Some((hand_res.ring.metacarpal, hand_res.ring.proximal))
+        }
+        PhysicsHandBone::RingProximal => {
+            return Some((hand_res.ring.proximal, hand_res.ring.intermediate))
+        }
+        PhysicsHandBone::RingIntermediate => {
+            return Some((hand_res.ring.intermediate, hand_res.ring.distal))
+        }
+        PhysicsHandBone::RingDistal => return Some((hand_res.ring.distal, hand_res.ring.tip)),
+        PhysicsHandBone::RingTip => return None,
+        PhysicsHandBone::LittleMetacarpal => {
+            return Some((hand_res.little.metacarpal, hand_res.little.proximal))
+        }
+        PhysicsHandBone::LittleProximal => {
+            return Some((hand_res.little.proximal, hand_res.little.intermediate))
+        }
+        PhysicsHandBone::LittleIntermediate => {
+            return Some((hand_res.little.intermediate, hand_res.little.distal))
+        }
+        PhysicsHandBone::LittleDistal => {
+            return Some((hand_res.little.distal, hand_res.little.tip))
+        }
+        PhysicsHandBone::LittleTip => return None,
+    };
+}
+
+fn get_hand_res(res: &Res<'_, HandsResource>, hand: Hand) -> HandResource {
+    match hand {
+        Hand::Left => res.left.clone(),
+        Hand::Right => res.right.clone(),
+    }
+}
+
+#[derive(Event, Default)]
+pub struct SpawnCubeRequest;
+
+#[derive(Resource)]
+pub struct SpawnCubeTimer(Timer);
+
+fn request_cube_spawn(
+    oculus_controller: Res<OculusController>,
+    frame_state: Res<XrFrameState>,
+    xr_input: Res<XrInput>,
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+    mut writer: EventWriter<SpawnCubeRequest>,
+    time: Res<Time>,
+    mut timer: ResMut<SpawnCubeTimer>,
+) {
+    timer.0.tick(time.delta());
+    if timer.0.finished() {
+        //lock frame
+        let frame_state = *frame_state.lock().unwrap();
+        //get controller
+        let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
+        //get controller triggers
+        let left_main_button = controller.a_button();
+        if left_main_button {
+            writer.send(SpawnCubeRequest::default());
+            timer.0.reset();
+        }
+        let right_main_button = controller.x_button();
+        if right_main_button {
+            writer.send(SpawnCubeRequest::default());
+            timer.0.reset();
+        }
+    }
+}
+
+fn cube_spawner(
+    mut commands: Commands,
+    mut meshes: ResMut<Assets<Mesh>>,
+    mut materials: ResMut<Assets<StandardMaterial>>,
+    mut events: EventReader<SpawnCubeRequest>,
+) {
+    for request in events.iter() {
+        // cube
+        commands.spawn((
+            PbrBundle {
+                mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
+                material: materials.add(Color::rgb(0.8, 0.7, 0.6).into()),
+                transform: Transform::from_xyz(0.0, 1.0, 0.0),
+                ..default()
+            },
+            RigidBody::Dynamic,
+            Collider::cuboid(0.05, 0.05, 0.05),
+            ColliderDebugColor(Color::hsl(220.0, 1.0, 0.3)),
+            XRInteractable,
+            XRInteractableState::default(),
+            Grabbable,
+            Touched(false),
+        ));
+    }
+}
+
+//TODO: find a real place for this
+fn prototype_interaction_input(
+    oculus_controller: Res<OculusController>,
+    frame_state: Res<XrFrameState>,
+    xr_input: Res<XrInput>,
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+    mut right_interactor_query: Query<
+        (&mut XRInteractorState),
+        (
+            With<XRDirectInteractor>,
+            With<OpenXRRightController>,
+            Without<OpenXRLeftController>,
+        ),
+    >,
+    mut left_interactor_query: Query<
+        (&mut XRInteractorState),
+        (
+            With<XRDirectInteractor>,
+            With<OpenXRLeftController>,
+            Without<OpenXRRightController>,
+        ),
+    >,
+) {
+    //lock frame
+    let frame_state = *frame_state.lock().unwrap();
+    //get controller
+    let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
+    //get controller triggers
+    let left_trigger = controller.trigger(Hand::Left);
+    let right_trigger = controller.trigger(Hand::Right);
+    //get the interactors and do state stuff
+    let mut left_state = left_interactor_query.single_mut();
+    if left_trigger > 0.8 {
+        *left_state = XRInteractorState::Selecting;
+    } else {
+        *left_state = XRInteractorState::Idle;
+    }
+    let mut right_state = right_interactor_query.single_mut();
+    if right_trigger > 0.8 {
+        *right_state = XRInteractorState::Selecting;
+    } else {
+        *right_state = XRInteractorState::Idle;
+    }
+}
+
+#[derive(Component)]
+pub struct Grabbable;
+
+pub fn update_grabbables(
+    mut events: EventReader<InteractionEvent>,
+    mut grabbable_query: Query<(
+        Entity,
+        &mut Transform,
+        With<Grabbable>,
+        Without<XRDirectInteractor>,
+        Option<&mut RigidBody>,
+    )>,
+    mut interactor_query: Query<(
+        &GlobalTransform,
+        &XRInteractorState,
+        &mut XRSelection,
+        Without<Grabbable>,
+    )>,
+) {
+    //so basically the idea is to try all the events?
+    for event in events.read() {
+        // info!("some event");
+        match grabbable_query.get_mut(event.interactable) {
+            Ok(mut grabbable_transform) => {
+                // info!("we got a grabbable");
+                //now we need the location of our interactor
+                match interactor_query.get_mut(event.interactor) {
+                    Ok(mut interactor_transform) => {
+                        match *interactor_transform.2 {
+                            XRSelection::Empty => {
+                                match interactor_transform.1 {
+                                    XRInteractorState::Idle => match grabbable_transform.4 {
+                                        Some(mut thing) => {
+                                            *thing = RigidBody::Dynamic;
+                                            *interactor_transform.2 = XRSelection::Empty;
+                                        }
+                                        None => (),
+                                    },
+                                    XRInteractorState::Selecting => {
+                                        // info!("its a direct interactor?");
+                                        match grabbable_transform.4 {
+                                            Some(mut thing) => {
+                                                *thing = RigidBody::KinematicPositionBased;
+                                                *interactor_transform.2 =
+                                                    XRSelection::Full(grabbable_transform.0);
+                                            }
+                                            None => (),
+                                        }
+                                        *grabbable_transform.1 =
+                                            interactor_transform.0.compute_transform();
+                                    }
+                                }
+                            }
+                            XRSelection::Full(ent) => {
+                                info!("nah bro we holding something");
+                                match grabbable_transform.0 == ent {
+                                    true => {
+                                        *grabbable_transform.1 =
+                                            interactor_transform.0.compute_transform();
+                                    }
+                                    false => {}
+                                }
+                                match interactor_transform.1 {
+                                    XRInteractorState::Idle => {
+                                        *interactor_transform.2 = XRSelection::Empty
+                                    }
+                                    XRInteractorState::Selecting => {}
+                                }
+                            }
+                        }
+                    }
+                    Err(_) => {
+                        // info!("not a direct interactor")
+                    }
+                }
+            }
+            Err(_) => {
+                // info!("not a grabbable?")
+            }
+        }
+    }
+}

examples/xr.rs

@@ -3,10 +3,10 @@ use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin};
 use bevy::prelude::*;
 use bevy::transform::components::Transform;
 use bevy_oxr::input::XrInput;
-use bevy_oxr::resources::{XrFrameState, XrInstance, XrSession};
+use bevy_oxr::resources::{XrFrameState, XrSession};
 
 use bevy_oxr::xr_input::actions::XrActionSets;
-use bevy_oxr::xr_input::hand::{HandInputDebugRenderer, OpenXrHandInput};
+use bevy_oxr::xr_input::hands::common::{HandInputDebugRenderer, OpenXrHandInput};
 use bevy_oxr::xr_input::interactions::{
     draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
     update_interactable_states, InteractionEvent, Touched, XRDirectInteractor, XRInteractable,

src/graphics/vulkan.rs

@@ -20,6 +20,8 @@ use crate::VIEW_TYPE;
 
 pub fn initialize_xr_graphics(
     window: Option<RawHandleWrapper>,
+    // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+    // any extension at some point
 ) -> anyhow::Result<(
     RenderDevice,
     RenderQueue,
@@ -37,6 +39,8 @@ pub fn initialize_xr_graphics(
     XrInput,
     XrViews,
     XrFrameState,
+    // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+    // any extension at some point
 )> {
     use wgpu_hal::{api::Vulkan as V, Api};
 
@@ -405,6 +409,8 @@ pub fn initialize_xr_graphics(
             should_render: true,
         })
         .into(),
+        // Horrible hack to get the Handtacking extension Loaded, Replace with good system to load
+        // any extension at some point
     ))
 }
 

src/lib.rs

@@ -6,6 +6,8 @@ pub mod xr_input;
 
 use std::sync::{Arc, Mutex};
 
+use crate::xr_input::hands::hand_tracking::DisableHandTracking;
+use crate::xr_input::oculus_touch::ActionSets;
 use bevy::app::PluginGroupBuilder;
 use bevy::ecs::system::SystemState;
 use bevy::prelude::*;
@@ -18,22 +20,26 @@ use bevy::window::{PresentMode, PrimaryWindow, RawHandleWrapper};
 use input::XrInput;
 use openxr as xr;
 use resources::*;
+use xr::FormFactor;
 use xr_input::controllers::XrControllerType;
-use xr_input::hand::HandInputSource;
+use xr_input::hands::emulated::HandEmulationPlugin;
-use xr_input::handtracking::HandTrackingTracker;
+use xr_input::hands::hand_tracking::{HandTrackingData, HandTrackingPlugin};
 use xr_input::OpenXrInput;
 
-use crate::xr_input::oculus_touch::ActionSets;
-
 const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
 
 pub const LEFT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(1208214591);
 pub const RIGHT_XR_TEXTURE_HANDLE: ManualTextureViewHandle = ManualTextureViewHandle(3383858418);
 
 /// Adds OpenXR support to an App
-#[derive(Default)]
 pub struct OpenXrPlugin;
 
+impl Default for OpenXrPlugin {
+    fn default() -> Self {
+        OpenXrPlugin
+    }
+}
+
 #[derive(Resource)]
 pub struct FutureXrResources(
     pub  Arc<
@@ -82,6 +88,7 @@ impl Plugin for OpenXrPlugin {
             views,
             frame_state,
         ) = graphics::initialize_xr_graphics(primary_window).unwrap();
+        // std::thread::sleep(Duration::from_secs(5));
         debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
         debug!("Configured wgpu adapter Features: {:#?}", device.features());
         let mut future_xr_resources_inner = future_xr_resources_wrapper.lock().unwrap();
@@ -147,12 +154,18 @@ impl Plugin for OpenXrPlugin {
                 .insert_resource(views.clone())
                 .insert_resource(frame_state.clone())
                 .insert_resource(action_sets.clone());
-            let hands = xr_instance.exts().ext_hand_tracking.is_some();
+            let hands = xr_instance.exts().ext_hand_tracking.is_some()
+                && xr_instance
+                    .supports_hand_tracking(
+                        xr_instance
+                            .system(FormFactor::HEAD_MOUNTED_DISPLAY)
+                            .unwrap(),
+                    )
+                    .is_ok_and(|v| v);
             if hands {
-                app.insert_resource(HandTrackingTracker::new(&session).unwrap());
+                app.insert_resource(HandTrackingData::new(&session).unwrap());
-                app.insert_resource(HandInputSource::OpenXr);
             } else {
-                app.insert_resource(HandInputSource::Emulated);
+                app.insert_resource(DisableHandTracking::Both);
             }
 
             let (left, right) = swapchain.get_render_views();
@@ -208,8 +221,10 @@ impl PluginGroup for DefaultXrPlugins {
             .build()
             .disable::<RenderPlugin>()
             .disable::<PipelinedRenderingPlugin>()
-            .add_before::<RenderPlugin, _>(OpenXrPlugin)
+            .add_before::<RenderPlugin, _>(OpenXrPlugin::default())
             .add_after::<OpenXrPlugin, _>(OpenXrInput::new(XrControllerType::OculusTouch))
+            .add(HandEmulationPlugin)
+            .add(HandTrackingPlugin)
             .set(WindowPlugin {
                 #[cfg(not(target_os = "android"))]
                 primary_window: Some(Window {

src/xr_input/debug_gizmos.rs

@@ -1,11 +1,12 @@
 use bevy::prelude::{
-    info, Color, Gizmos, GlobalTransform, Plugin, Quat, Query, Res, Transform, Update, Vec2, Vec3,
+    Color, Gizmos, GlobalTransform, Plugin, Quat, Query, Res, Transform, Update, Vec2, Vec3, With,
-    With, Without,
+    Without,
 };
+use bevy::log::info;
 
 use crate::{
     input::XrInput,
-    resources::{XrFrameState, XrInstance, XrSession},
+    resources::{XrFrameState, XrSession},
 };
 
 use crate::xr_input::{
@@ -15,9 +16,7 @@ use crate::xr_input::{
 
 use super::{
     actions::XrActionSets,
-    handtracking::{HandTrackingRef, HandTrackingTracker},
     trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
-    QuatConv,
 };
 
 /// add debug renderer for controllers
@@ -35,7 +34,6 @@ pub fn draw_gizmos(
     oculus_controller: Res<OculusController>,
     frame_state: Res<XrFrameState>,
     xr_input: Res<XrInput>,
-    instance: Res<XrInstance>,
     session: Res<XrSession>,
     tracking_root_query: Query<(
         &mut Transform,
@@ -55,49 +53,38 @@ pub fn draw_gizmos(
         Without<OpenXRLeftController>,
         Without<OpenXRTrackingRoot>,
     )>,
-    hand_tracking: Option<Res<HandTrackingTracker>>,
     action_sets: Res<XrActionSets>,
 ) {
-    if let Some(hand_tracking) = hand_tracking {
+    // if let Some(hand_tracking) = hand_tracking {
-        let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
+    //     let handtracking_ref = hand_tracking.get_ref(&xr_input, &frame_state);
-        if let Some(joints) = handtracking_ref.get_left_poses() {
+    //     if let Some(joints) = handtracking_ref.get_poses(Hand::Left) {
-            for joint in joints {
+    //         for joint in joints.inner() {
-                let p = joint.pose.position;
+    //             let trans = Transform::from_rotation(joint.orientation);
-                let r = joint.pose.orientation;
+    //             gizmos.circle(
-                let quat = r.to_quat();
+    //                 joint.position,
-                let trans = Transform::from_rotation(quat);
+    //                 trans.forward(),
-                gizmos.circle(
+    //                 joint.radius,
-                    (p.x, p.y, p.z).into(),
+    //                 Color::ORANGE_RED,
-                    trans.forward(),
+    //             );
-                    joint.radius,
+    //         }
-                    Color::ORANGE_RED,
+    //     }
-                );
+    //     if let Some(joints) = handtracking_ref.get_poses(Hand::Right) {
-            }
+    //         for joint in joints.inner() {
-        } else {
+    //             let trans = Transform::from_rotation(joint.orientation);
-            info!("left_hand_poses returned None");
+    //             gizmos.circle(
-        }
+    //                 joint.position,
-        if let Some(joints) = handtracking_ref.get_right_poses() {
+    //                 trans.forward(),
-            for joint in joints {
+    //                 joint.radius,
-                let p = joint.pose.position;
+    //                 Color::LIME_GREEN,
-                let r = joint.pose.orientation;
+    //             );
-                let quat = r.to_quat();
+    //         }
-                let trans = Transform::from_rotation(quat);
+    //         return;
-                gizmos.circle(
+    //     }
-                    (p.x, p.y, p.z).into(),
+    // }
-                    trans.forward(),
-                    joint.radius,
-                    Color::LIME_GREEN,
-                );
-            }
-            return;
-        }
-    }
     //lock frame
     let frame_state = *frame_state.lock().unwrap();
     //get controller
     let controller = oculus_controller.get_ref(&session, &frame_state, &xr_input, &action_sets);
-    //tracking root?
-    let mut tracking_transform = &Transform::IDENTITY;
     let root = tracking_root_query.get_single();
     match root {
         Ok(position) => {
@@ -112,7 +99,6 @@ pub fn draw_gizmos(
                 0.2,
                 Color::RED,
             );
-            tracking_transform = position.0;
         }
         Err(_) => info!("too many tracking roots"),
     }

src/xr_input/hands/common.rs

@@ -0,0 +1,286 @@
+use bevy::prelude::{
+    default, Color, Commands, Component, Deref, DerefMut, Entity, Gizmos, Plugin, PostUpdate,
+    Query, Resource, SpatialBundle, Startup, Transform,
+};
+
+use crate::xr_input::{Hand, trackers::OpenXRTracker};
+
+use super::{HandBone, BoneTrackingStatus};
+
+/// add debug renderer for controllers
+#[derive(Default)]
+pub struct OpenXrHandInput;
+
+impl Plugin for OpenXrHandInput {
+    fn build(&self, app: &mut bevy::prelude::App) {
+        app.add_systems(Startup, spawn_hand_entities);
+    }
+}
+
+/// add debug renderer for controllers
+#[derive(Default)]
+pub struct HandInputDebugRenderer;
+
+impl Plugin for HandInputDebugRenderer {
+    fn build(&self, app: &mut bevy::prelude::App) {
+        app.add_systems(PostUpdate, draw_hand_entities);
+    }
+}
+
+#[derive(Resource, Default, Clone, Copy)]
+pub struct HandsResource {
+    pub left: HandResource,
+    pub right: HandResource,
+}
+#[derive(Clone, Copy)]
+pub struct HandResource {
+    pub palm: Entity,
+    pub wrist: Entity,
+    pub thumb: ThumbResource,
+    pub index: IndexResource,
+    pub middle: MiddleResource,
+    pub ring: RingResource,
+    pub little: LittleResource,
+}
+
+impl Default for HandResource {
+    fn default() -> Self {
+        Self {
+            palm: Entity::PLACEHOLDER,
+            wrist: Entity::PLACEHOLDER,
+            thumb: Default::default(),
+            index: Default::default(),
+            middle: Default::default(),
+            ring: Default::default(),
+            little: Default::default(),
+        }
+    }
+}
+#[derive(Clone, Copy)]
+pub struct ThumbResource {
+    pub metacarpal: Entity,
+    pub proximal: Entity,
+    pub distal: Entity,
+    pub tip: Entity,
+}
+
+impl Default for ThumbResource {
+    fn default() -> Self {
+        Self {
+            metacarpal: Entity::PLACEHOLDER,
+            proximal: Entity::PLACEHOLDER,
+            distal: Entity::PLACEHOLDER,
+            tip: Entity::PLACEHOLDER,
+        }
+    }
+}
+#[derive(Clone, Copy)]
+pub struct IndexResource {
+    pub metacarpal: Entity,
+    pub proximal: Entity,
+    pub intermediate: Entity,
+    pub distal: Entity,
+    pub tip: Entity,
+}
+
+impl Default for IndexResource {
+    fn default() -> Self {
+        Self {
+            metacarpal: Entity::PLACEHOLDER,
+            proximal: Entity::PLACEHOLDER,
+            intermediate: Entity::PLACEHOLDER,
+            distal: Entity::PLACEHOLDER,
+            tip: Entity::PLACEHOLDER,
+        }
+    }
+}
+#[derive(Clone, Copy)]
+pub struct MiddleResource {
+    pub metacarpal: Entity,
+    pub proximal: Entity,
+    pub intermediate: Entity,
+    pub distal: Entity,
+    pub tip: Entity,
+}
+impl Default for MiddleResource {
+    fn default() -> Self {
+        Self {
+            metacarpal: Entity::PLACEHOLDER,
+            proximal: Entity::PLACEHOLDER,
+            intermediate: Entity::PLACEHOLDER,
+            distal: Entity::PLACEHOLDER,
+            tip: Entity::PLACEHOLDER,
+        }
+    }
+}
+#[derive(Clone, Copy)]
+pub struct RingResource {
+    pub metacarpal: Entity,
+    pub proximal: Entity,
+    pub intermediate: Entity,
+    pub distal: Entity,
+    pub tip: Entity,
+}
+impl Default for RingResource {
+    fn default() -> Self {
+        Self {
+            metacarpal: Entity::PLACEHOLDER,
+            proximal: Entity::PLACEHOLDER,
+            intermediate: Entity::PLACEHOLDER,
+            distal: Entity::PLACEHOLDER,
+            tip: Entity::PLACEHOLDER,
+        }
+    }
+}
+#[derive(Clone, Copy)]
+pub struct LittleResource {
+    pub metacarpal: Entity,
+    pub proximal: Entity,
+    pub intermediate: Entity,
+    pub distal: Entity,
+    pub tip: Entity,
+}
+impl Default for LittleResource {
+    fn default() -> Self {
+        Self {
+            metacarpal: Entity::PLACEHOLDER,
+            proximal: Entity::PLACEHOLDER,
+            intermediate: Entity::PLACEHOLDER,
+            distal: Entity::PLACEHOLDER,
+            tip: Entity::PLACEHOLDER,
+        }
+    }
+}
+
+pub fn spawn_hand_entities(mut commands: Commands) {
+    let hands = [Hand::Left, Hand::Right];
+    let bones = HandBone::get_all_bones();
+    //hand resource
+    let mut hand_resource = HandsResource { ..default() };
+    for hand in hands.iter() {
+        for bone in bones.iter() {
+            let boneid = commands
+                .spawn((
+                    SpatialBundle::default(),
+                    bone.clone(),
+                    OpenXRTracker,
+                    hand.clone(),
+                    BoneTrackingStatus::Emulated,
+                    HandBoneRadius(0.1),
+                ))
+                .id();
+            match hand {
+                Hand::Left => match bone {
+                    HandBone::Palm => hand_resource.left.palm = boneid,
+                    HandBone::Wrist => hand_resource.left.wrist = boneid,
+                    HandBone::ThumbMetacarpal => hand_resource.left.thumb.metacarpal = boneid,
+                    HandBone::ThumbProximal => hand_resource.left.thumb.proximal = boneid,
+                    HandBone::ThumbDistal => hand_resource.left.thumb.distal = boneid,
+                    HandBone::ThumbTip => hand_resource.left.thumb.tip = boneid,
+                    HandBone::IndexMetacarpal => hand_resource.left.index.metacarpal = boneid,
+                    HandBone::IndexProximal => hand_resource.left.index.proximal = boneid,
+                    HandBone::IndexIntermediate => hand_resource.left.index.intermediate = boneid,
+                    HandBone::IndexDistal => hand_resource.left.index.distal = boneid,
+                    HandBone::IndexTip => hand_resource.left.index.tip = boneid,
+                    HandBone::MiddleMetacarpal => hand_resource.left.middle.metacarpal = boneid,
+                    HandBone::MiddleProximal => hand_resource.left.middle.proximal = boneid,
+                    HandBone::MiddleIntermediate => hand_resource.left.middle.intermediate = boneid,
+                    HandBone::MiddleDistal => hand_resource.left.middle.distal = boneid,
+                    HandBone::MiddleTip => hand_resource.left.middle.tip = boneid,
+                    HandBone::RingMetacarpal => hand_resource.left.ring.metacarpal = boneid,
+                    HandBone::RingProximal => hand_resource.left.ring.proximal = boneid,
+                    HandBone::RingIntermediate => hand_resource.left.ring.intermediate = boneid,
+                    HandBone::RingDistal => hand_resource.left.ring.distal = boneid,
+                    HandBone::RingTip => hand_resource.left.ring.tip = boneid,
+                    HandBone::LittleMetacarpal => hand_resource.left.little.metacarpal = boneid,
+                    HandBone::LittleProximal => hand_resource.left.little.proximal = boneid,
+                    HandBone::LittleIntermediate => hand_resource.left.little.intermediate = boneid,
+                    HandBone::LittleDistal => hand_resource.left.little.distal = boneid,
+                    HandBone::LittleTip => hand_resource.left.little.tip = boneid,
+                },
+                Hand::Right => match bone {
+                    HandBone::Palm => hand_resource.right.palm = boneid,
+                    HandBone::Wrist => hand_resource.right.wrist = boneid,
+                    HandBone::ThumbMetacarpal => hand_resource.right.thumb.metacarpal = boneid,
+                    HandBone::ThumbProximal => hand_resource.right.thumb.proximal = boneid,
+                    HandBone::ThumbDistal => hand_resource.right.thumb.distal = boneid,
+                    HandBone::ThumbTip => hand_resource.right.thumb.tip = boneid,
+                    HandBone::IndexMetacarpal => hand_resource.right.index.metacarpal = boneid,
+                    HandBone::IndexProximal => hand_resource.right.index.proximal = boneid,
+                    HandBone::IndexIntermediate => hand_resource.right.index.intermediate = boneid,
+                    HandBone::IndexDistal => hand_resource.right.index.distal = boneid,
+                    HandBone::IndexTip => hand_resource.right.index.tip = boneid,
+                    HandBone::MiddleMetacarpal => hand_resource.right.middle.metacarpal = boneid,
+                    HandBone::MiddleProximal => hand_resource.right.middle.proximal = boneid,
+                    HandBone::MiddleIntermediate => {
+                        hand_resource.right.middle.intermediate = boneid
+                    }
+                    HandBone::MiddleDistal => hand_resource.right.middle.distal = boneid,
+                    HandBone::MiddleTip => hand_resource.right.middle.tip = boneid,
+                    HandBone::RingMetacarpal => hand_resource.right.ring.metacarpal = boneid,
+                    HandBone::RingProximal => hand_resource.right.ring.proximal = boneid,
+                    HandBone::RingIntermediate => hand_resource.right.ring.intermediate = boneid,
+                    HandBone::RingDistal => hand_resource.right.ring.distal = boneid,
+                    HandBone::RingTip => hand_resource.right.ring.tip = boneid,
+                    HandBone::LittleMetacarpal => hand_resource.right.little.metacarpal = boneid,
+                    HandBone::LittleProximal => hand_resource.right.little.proximal = boneid,
+                    HandBone::LittleIntermediate => {
+                        hand_resource.right.little.intermediate = boneid
+                    }
+                    HandBone::LittleDistal => hand_resource.right.little.distal = boneid,
+                    HandBone::LittleTip => hand_resource.right.little.tip = boneid,
+                },
+            }
+        }
+    }
+    commands.insert_resource(hand_resource);
+}
+
+#[derive(Debug, Component, DerefMut, Deref)]
+pub struct HandBoneRadius(pub f32);
+
+pub fn draw_hand_entities(
+    mut gizmos: Gizmos,
+    query: Query<(&Transform, &HandBone, &HandBoneRadius)>,
+) {
+    for (transform, hand_bone, hand_bone_radius) in query.iter() {
+        let (_, color) = get_bone_gizmo_style(hand_bone);
+        gizmos.sphere(
+            transform.translation,
+            transform.rotation,
+            hand_bone_radius.0,
+            color,
+        );
+    }
+}
+
+pub(crate) fn get_bone_gizmo_style(hand_bone: &HandBone) -> (f32, Color) {
+    match hand_bone {
+        HandBone::Palm => (0.01, Color::WHITE),
+        HandBone::Wrist => (0.01, Color::GRAY),
+        HandBone::ThumbMetacarpal => (0.01, Color::RED),
+        HandBone::ThumbProximal => (0.008, Color::RED),
+        HandBone::ThumbDistal => (0.006, Color::RED),
+        HandBone::ThumbTip => (0.004, Color::RED),
+        HandBone::IndexMetacarpal => (0.01, Color::ORANGE),
+        HandBone::IndexProximal => (0.008, Color::ORANGE),
+        HandBone::IndexIntermediate => (0.006, Color::ORANGE),
+        HandBone::IndexDistal => (0.004, Color::ORANGE),
+        HandBone::IndexTip => (0.002, Color::ORANGE),
+        HandBone::MiddleMetacarpal => (0.01, Color::YELLOW),
+        HandBone::MiddleProximal => (0.008, Color::YELLOW),
+        HandBone::MiddleIntermediate => (0.006, Color::YELLOW),
+        HandBone::MiddleDistal => (0.004, Color::YELLOW),
+        HandBone::MiddleTip => (0.002, Color::YELLOW),
+        HandBone::RingMetacarpal => (0.01, Color::GREEN),
+        HandBone::RingProximal => (0.008, Color::GREEN),
+        HandBone::RingIntermediate => (0.006, Color::GREEN),
+        HandBone::RingDistal => (0.004, Color::GREEN),
+        HandBone::RingTip => (0.002, Color::GREEN),
+        HandBone::LittleMetacarpal => (0.01, Color::BLUE),
+        HandBone::LittleProximal => (0.008, Color::BLUE),
+        HandBone::LittleIntermediate => (0.006, Color::BLUE),
+        HandBone::LittleDistal => (0.004, Color::BLUE),
+        HandBone::LittleTip => (0.002, Color::BLUE),
+    }
+}

src/xr_input/hands/emulated.rs

@@ -0,0 +1,528 @@
+use std::f32::consts::PI;
+
+use bevy::prelude::*;
+use openxr::{ActionTy, HandJoint};
+
+use super::common::{get_bone_gizmo_style, HandBoneRadius};
+use crate::{
+    resources::{XrInstance, XrSession},
+    xr_input::{
+        actions::{
+            ActionHandednes, ActionType, SetupActionSet, SetupActionSets, XrActionSets, XrBinding,
+        },
+        hand_poses::get_simulated_open_hand_transforms,
+        trackers::{OpenXRLeftController, OpenXRRightController, OpenXRTrackingRoot},
+        Hand,
+    },
+};
+
+use super::{BoneTrackingStatus, HandBone};
+
+pub enum TouchValue<T: ActionTy> {
+    None,
+    Touched(T),
+}
+
+pub struct HandEmulationPlugin;
+
+impl Plugin for HandEmulationPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(PreUpdate, update_hand_skeleton_from_emulated);
+        app.add_systems(Startup, setup_hand_emulation_action_set);
+    }
+}
+
+const HAND_ACTION_SET: &str = "hand_pose_approx";
+
+fn setup_hand_emulation_action_set(mut action_sets: ResMut<SetupActionSets>) {
+    let mut action_set = action_sets.add_action_set(HAND_ACTION_SET, "Hand Pose Approximaiton", 0);
+    action_set.new_action(
+        "thumb_touch",
+        "Thumb Touched",
+        ActionType::Bool,
+        ActionHandednes::Double,
+    );
+    action_set.new_action(
+        "thumb_x",
+        "Thumb X",
+        ActionType::F32,
+        ActionHandednes::Double,
+    );
+    action_set.new_action(
+        "thumb_y",
+        "Thumb Y",
+        ActionType::F32,
+        ActionHandednes::Double,
+    );
+
+    action_set.new_action(
+        "index_touch",
+        "Index Finger Touched",
+        ActionType::Bool,
+        ActionHandednes::Double,
+    );
+    action_set.new_action(
+        "index_value",
+        "Index Finger Pull",
+        ActionType::F32,
+        ActionHandednes::Double,
+    );
+
+    action_set.new_action(
+        "middle_value",
+        "Middle Finger Pull",
+        ActionType::F32,
+        ActionHandednes::Double,
+    );
+    action_set.new_action(
+        "ring_value",
+        "Ring Finger Pull",
+        ActionType::F32,
+        ActionHandednes::Double,
+    );
+    action_set.new_action(
+        "little_value",
+        "Little Finger Pull",
+        ActionType::F32,
+        ActionHandednes::Double,
+    );
+
+    suggest_oculus_touch_profile(&mut action_set);
+}
+
+pub struct EmulatedHandPoseData {}
+
+fn suggest_oculus_touch_profile(action_set: &mut SetupActionSet) {
+    action_set.suggest_binding(
+        "/interaction_profiles/oculus/touch_controller",
+        &[
+            XrBinding::new("thumb_x", "/user/hand/left/input/thumbstick/x"),
+            XrBinding::new("thumb_x", "/user/hand/right/input/thumbstick/x"),
+            XrBinding::new("thumb_y", "/user/hand/left/input/thumbstick/y"),
+            XrBinding::new("thumb_y", "/user/hand/right/input/thumbstick/y"),
+            XrBinding::new("thumb_touch", "/user/hand/left/input/thumbstick/touch"),
+            XrBinding::new("thumb_touch", "/user/hand/right/input/thumbstick/touch"),
+            XrBinding::new("thumb_touch", "/user/hand/left/input/x/touch"),
+            XrBinding::new("thumb_touch", "/user/hand/left/input/y/touch"),
+            XrBinding::new("thumb_touch", "/user/hand/right/input/a/touch"),
+            XrBinding::new("thumb_touch", "/user/hand/right/input/b/touch"),
+            XrBinding::new("thumb_touch", "/user/hand/left/input/thumbrest/touch"),
+            XrBinding::new("thumb_touch", "/user/hand/right/input/thumbrest/touch"),
+            XrBinding::new("index_touch", "/user/hand/left/input/trigger/touch"),
+            XrBinding::new("index_value", "/user/hand/left/input/trigger/value"),
+            XrBinding::new("index_touch", "/user/hand/right/input/trigger/touch"),
+            XrBinding::new("index_value", "/user/hand/right/input/trigger/value"),
+            XrBinding::new("middle_value", "/user/hand/left/input/squeeze/value"),
+            XrBinding::new("middle_value", "/user/hand/right/input/squeeze/value"),
+            XrBinding::new("ring_value", "/user/hand/left/input/squeeze/value"),
+            XrBinding::new("ring_value", "/user/hand/right/input/squeeze/value"),
+            XrBinding::new("little_value", "/user/hand/left/input/squeeze/value"),
+            XrBinding::new("little_value", "/user/hand/right/input/squeeze/value"),
+        ],
+    );
+}
+
+pub(crate) fn update_hand_skeleton_from_emulated(
+    session: Res<XrSession>,
+    instance: Res<XrInstance>,
+    action_sets: Res<XrActionSets>,
+    left_controller_transform: Query<&Transform, With<OpenXRLeftController>>,
+    right_controller_transform: Query<&Transform, With<OpenXRRightController>>,
+    tracking_root_transform: Query<&Transform, With<OpenXRTrackingRoot>>,
+    mut bones: Query<
+        (
+            &mut Transform,
+            &HandBone,
+            &Hand,
+            &BoneTrackingStatus,
+            &mut HandBoneRadius,
+        ),
+        (
+            Without<OpenXRLeftController>,
+            Without<OpenXRRightController>,
+            Without<OpenXRTrackingRoot>,
+        ),
+    >,
+) {
+    let mut data: [[Transform; 26]; 2] = [[Transform::default(); 26]; 2];
+    for (subaction_path, hand) in [
+        (
+            instance.string_to_path("/user/hand/left").unwrap(),
+            Hand::Left,
+        ),
+        (
+            instance.string_to_path("/user/hand/right").unwrap(),
+            Hand::Right,
+        ),
+    ] {
+        let thumb_curl = match action_sets
+            .get_action_bool(HAND_ACTION_SET, "thumb_touch")
+            .unwrap()
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state
+        {
+            true => 1.0,
+            false => 0.0,
+        };
+        let index_curl = action_sets
+            .get_action_f32(HAND_ACTION_SET, "index_value")
+            .unwrap()
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        let middle_curl = action_sets
+            .get_action_f32(HAND_ACTION_SET, "middle_value")
+            .unwrap()
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        let ring_curl = action_sets
+            .get_action_f32(HAND_ACTION_SET, "ring_value")
+            .unwrap()
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        let little_curl = action_sets
+            .get_action_f32(HAND_ACTION_SET, "little_value")
+            .unwrap()
+            .state(&session, subaction_path)
+            .unwrap()
+            .current_state;
+        data[match hand {
+            Hand::Left => 0,
+            Hand::Right => 1,
+        }] = update_hand_bones_emulated(
+            match hand {
+                Hand::Left => left_controller_transform.single(),
+                Hand::Right => right_controller_transform.single(),
+            },
+            hand,
+            thumb_curl,
+            index_curl,
+            middle_curl,
+            ring_curl,
+            little_curl,
+        );
+    }
+    let trt = tracking_root_transform.single();
+    for (mut t, bone, hand, status, mut radius) in bones.iter_mut() {
+        match status {
+            BoneTrackingStatus::Emulated => {}
+            BoneTrackingStatus::Tracked => continue,
+        }
+        radius.0 = get_bone_gizmo_style(bone).0;
+
+        *t = data[match hand {
+            Hand::Left => 0,
+            Hand::Right => 1,
+        }][bone.get_index_from_bone()];
+        *t = t.with_scale(trt.scale);
+        *t = t.with_rotation(trt.rotation * t.rotation);
+        *t = t.with_translation(trt.transform_point(t.translation));
+    }
+}
+pub fn update_hand_bones_emulated(
+    controller_transform: &Transform,
+    hand: Hand,
+    thumb_curl: f32,
+    index_curl: f32,
+    middle_curl: f32,
+    ring_curl: f32,
+    little_curl: f32,
+) -> [Transform; 26] {
+    let left_hand_rot = Quat::from_rotation_y(PI);
+    let hand_translation: Vec3 = controller_transform.translation;
+
+    let controller_quat: Quat = match hand {
+        Hand::Left => controller_transform.rotation.mul_quat(left_hand_rot),
+        Hand::Right => controller_transform.rotation,
+    };
+
+    let splay_direction = match hand {
+        Hand::Left => -1.0,
+        Hand::Right => 1.0,
+    };
+    //lets make a structure to hold our calculated transforms for now
+    let mut calc_transforms = [Transform::default(); 26];
+
+    //get palm quat
+    let y = Quat::from_rotation_y(-90.0 * PI / 180.0);
+    let x = Quat::from_rotation_x(-90.0 * PI / 180.0);
+    let palm_quat = controller_quat.mul_quat(y).mul_quat(x);
+    //get simulated bones
+    let hand_transform_array: [Transform; 26] = get_simulated_open_hand_transforms(hand);
+    //palm
+    let palm = hand_transform_array[HandJoint::PALM];
+    calc_transforms[HandJoint::PALM] = Transform {
+        translation: hand_translation + palm.translation,
+        ..default()
+    };
+    //wrist
+    let wrist = hand_transform_array[HandJoint::WRIST];
+    calc_transforms[HandJoint::WRIST] = Transform {
+        translation: hand_translation + palm.translation + palm_quat.mul_vec3(wrist.translation),
+        ..default()
+    };
+
+    //thumb
+    let thumb_joints = [
+        HandJoint::THUMB_METACARPAL,
+        HandJoint::THUMB_PROXIMAL,
+        HandJoint::THUMB_DISTAL,
+        HandJoint::THUMB_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * 30.0 * PI / 180.0);
+    let huh = Quat::from_rotation_x(-35.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(huh).mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, thumb_curl);
+                let tp_lrot = Quat::from_rotation_y(splay_direction * curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+
+    //index
+    let thumb_joints = [
+        HandJoint::INDEX_METACARPAL,
+        HandJoint::INDEX_PROXIMAL,
+        HandJoint::INDEX_INTERMEDIATE,
+        HandJoint::INDEX_DISTAL,
+        HandJoint::INDEX_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * 10.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, index_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+
+    //middle
+    let thumb_joints = [
+        HandJoint::MIDDLE_METACARPAL,
+        HandJoint::MIDDLE_PROXIMAL,
+        HandJoint::MIDDLE_INTERMEDIATE,
+        HandJoint::MIDDLE_DISTAL,
+        HandJoint::MIDDLE_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * 0.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, middle_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+    //ring
+    let thumb_joints = [
+        HandJoint::RING_METACARPAL,
+        HandJoint::RING_PROXIMAL,
+        HandJoint::RING_INTERMEDIATE,
+        HandJoint::RING_DISTAL,
+        HandJoint::RING_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * -10.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, ring_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+
+    //little
+    let thumb_joints = [
+        HandJoint::LITTLE_METACARPAL,
+        HandJoint::LITTLE_PROXIMAL,
+        HandJoint::LITTLE_INTERMEDIATE,
+        HandJoint::LITTLE_DISTAL,
+        HandJoint::LITTLE_TIP,
+    ];
+    let mut prior_start: Option<Vec3> = None;
+    let mut prior_quat: Option<Quat> = None;
+    let mut prior_vector: Option<Vec3> = None;
+    let splay = Quat::from_rotation_y(splay_direction * -20.0 * PI / 180.0);
+    let splay_quat = palm_quat.mul_quat(splay);
+    for bone in thumb_joints.iter() {
+        match prior_start {
+            Some(start) => {
+                let curl_angle: f32 = get_bone_curl_angle(*bone, little_curl);
+                let tp_lrot = Quat::from_rotation_x(curl_angle * PI / 180.0);
+                let tp_quat = prior_quat.unwrap().mul_quat(tp_lrot);
+                let thumb_prox = hand_transform_array[*bone];
+                let tp_start = start + prior_vector.unwrap();
+                let tp_vector = tp_quat.mul_vec3(thumb_prox.translation);
+                prior_start = Some(tp_start);
+                prior_quat = Some(tp_quat);
+                prior_vector = Some(tp_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tp_start + tp_vector,
+                    ..default()
+                };
+            }
+            None => {
+                let thumb_meta = hand_transform_array[*bone];
+                let tm_start = hand_translation
+                    + palm_quat.mul_vec3(palm.translation)
+                    + palm_quat.mul_vec3(wrist.translation);
+                let tm_vector = palm_quat.mul_vec3(thumb_meta.translation);
+                prior_start = Some(tm_start);
+                prior_quat = Some(splay_quat);
+                prior_vector = Some(tm_vector);
+                //store it
+                calc_transforms[*bone] = Transform {
+                    translation: tm_start + tm_vector,
+                    ..default()
+                };
+            }
+        }
+    }
+    calc_transforms
+}
+
+fn get_bone_curl_angle(bone: HandJoint, curl: f32) -> f32 {
+    let mul: f32 = match bone {
+        HandJoint::INDEX_PROXIMAL => 0.0,
+        HandJoint::MIDDLE_PROXIMAL => 0.0,
+        HandJoint::RING_PROXIMAL => 0.0,
+        HandJoint::LITTLE_PROXIMAL => 0.0,
+        HandJoint::THUMB_PROXIMAL => 0.0,
+        HandJoint::THUMB_TIP => 0.1,
+        HandJoint::THUMB_DISTAL => 0.1,
+        HandJoint::THUMB_METACARPAL => 0.1,
+        _ => 1.0,
+    };
+    let curl_angle = -((mul * curl * 80.0) + 5.0);
+    return curl_angle;
+}

src/xr_input/hands/hand_tracking.rs

@@ -0,0 +1,211 @@
+use bevy::prelude::*;
+use openxr::{HandTracker, Result, SpaceLocationFlags};
+
+use crate::{
+    input::XrInput,
+
+    resources::{XrFrameState, XrSession},
+    xr_input::{
+         hands::HandBone, trackers::OpenXRTrackingRoot, Hand, QuatConv,
+        Vec3Conv,
+    },
+};
+use super::common::HandBoneRadius;
+
+use super::BoneTrackingStatus;
+
+#[derive(Resource, PartialEq)]
+pub enum DisableHandTracking {
+    OnlyLeft,
+    OnlyRight,
+    Both,
+}
+pub struct HandTrackingPlugin;
+
+#[derive(Resource)]
+pub struct HandTrackingData {
+    left_hand: HandTracker,
+    right_hand: HandTracker,
+}
+
+impl HandTrackingData {
+    pub fn new(session: &XrSession) -> Result<HandTrackingData> {
+        let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
+        let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
+        Ok(HandTrackingData {
+            left_hand: left,
+            right_hand: right,
+        })
+    }
+    pub fn get_ref<'a>(
+        &'a self,
+        input: &'a XrInput,
+        frame_state: &'a XrFrameState,
+    ) -> HandTrackingRef<'a> {
+        HandTrackingRef {
+            tracking: self,
+            input,
+            frame_state,
+        }
+    }
+}
+
+pub struct HandTrackingRef<'a> {
+    tracking: &'a HandTrackingData,
+    input: &'a XrInput,
+    frame_state: &'a XrFrameState,
+}
+#[derive(Debug)]
+pub struct HandJoint {
+    pub position: Vec3,
+    pub position_valid: bool,
+    pub position_tracked: bool,
+    pub orientation: Quat,
+    pub orientation_valid: bool,
+    pub orientation_tracked: bool,
+    pub radius: f32,
+}
+
+pub struct HandJoints {
+    inner: [HandJoint; 26],
+}
+impl HandJoints {
+    pub fn inner(&self) -> &[HandJoint; 26] {
+        &self.inner
+    }
+}
+
+impl HandJoints {
+    pub fn get_joint(&self, bone: HandBone) -> &HandJoint {
+        &self.inner[bone.get_index_from_bone()]
+    }
+}
+
+impl<'a> HandTrackingRef<'a> {
+    pub fn get_poses(&self, side: Hand) -> Option<HandJoints> {
+        self.input
+            .stage
+            .locate_hand_joints(
+                match side {
+                    Hand::Left => &self.tracking.left_hand,
+                    Hand::Right => &self.tracking.right_hand,
+                },
+                self.frame_state.lock().unwrap().predicted_display_time,
+            )
+            .unwrap()
+            .map(|joints| {
+                joints
+                    .into_iter()
+                    .map(|joint| HandJoint {
+                        position: joint.pose.position.to_vec3(),
+                        orientation: joint.pose.orientation.to_quat(),
+                        position_valid: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::POSITION_VALID),
+                        position_tracked: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::POSITION_TRACKED),
+                        orientation_valid: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::ORIENTATION_VALID),
+                        orientation_tracked: joint
+                            .location_flags
+                            .contains(SpaceLocationFlags::ORIENTATION_TRACKED),
+                        radius: joint.radius,
+                    })
+                    .collect::<Vec<HandJoint>>()
+                    .try_into()
+                    .unwrap()
+            })
+            .map(|joints| HandJoints { inner: joints })
+    }
+}
+
+impl Plugin for HandTrackingPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(
+            PreUpdate,
+            (
+                update_hand_bones.run_if(|dh: Option<Res<DisableHandTracking>>| {
+                    !dh.is_some_and(|v| *v == DisableHandTracking::Both)
+                }),
+                update_tracking_state_on_disable,
+            ),
+        );
+    }
+}
+
+fn update_tracking_state_on_disable(
+    mut is_off: Local<bool>,
+    disabled_tracking: Option<Res<DisableHandTracking>>,
+    mut tracking_states: Query<&mut BoneTrackingStatus>,
+) {
+    if !*is_off
+        && disabled_tracking
+            .as_ref()
+            .is_some_and(|t| **t == DisableHandTracking::Both)
+    {
+        tracking_states
+            .par_iter_mut()
+            .for_each(|mut state| *state = BoneTrackingStatus::Emulated);
+    }
+    *is_off = disabled_tracking
+        .as_ref()
+        .is_some_and(|t| **t == DisableHandTracking::Both);
+}
+
+pub fn update_hand_bones(
+    disabled_tracking: Option<Res<DisableHandTracking>>,
+    hand_tracking: Option<Res<HandTrackingData>>,
+    xr_input: Res<XrInput>,
+    xr_frame_state: Res<XrFrameState>,
+    root_query: Query<(&Transform, With<OpenXRTrackingRoot>, Without<HandBone>)>,
+    mut bones: Query<(
+        &mut Transform,
+        &Hand,
+        &HandBone,
+        &mut HandBoneRadius,
+        &mut BoneTrackingStatus,
+    )>,
+) {
+    let hand_ref = match hand_tracking.as_ref() {
+        Some(h) => h.get_ref(&xr_input, &xr_frame_state),
+        None => {
+            warn!("No Handtracking data!");
+            return;
+        }
+    };
+    let (root_transform, _, _) = root_query.get_single().unwrap();
+    let left_hand_data = hand_ref.get_poses(Hand::Left);
+    let right_hand_data = hand_ref.get_poses(Hand::Right);
+    bones
+        .par_iter_mut()
+        .for_each(|(mut transform, hand, bone, mut radius, mut status)| {
+            match (&hand, disabled_tracking.as_ref().map(|d| d.as_ref())) {
+                (Hand::Left, Some(DisableHandTracking::OnlyLeft)) => {
+                    *status = BoneTrackingStatus::Emulated;
+                    return;
+                }
+                (Hand::Right, Some(DisableHandTracking::OnlyRight)) => {
+                    *status = BoneTrackingStatus::Emulated;
+                    return;
+                }
+                _ => {}
+            }
+            let bone_data = match (hand, &left_hand_data, &right_hand_data) {
+                (Hand::Left, Some(data), _) => data.get_joint(*bone),
+                (Hand::Right, _, Some(data)) => data.get_joint(*bone),
+                _ => {
+                    *status = BoneTrackingStatus::Emulated;
+                    return;
+                }
+            };
+            if *status == BoneTrackingStatus::Emulated {
+                *status = BoneTrackingStatus::Tracked;
+            }
+            radius.0 = bone_data.radius;
+            *transform = transform
+                .with_translation(root_transform.transform_point(bone_data.position))
+                .with_rotation(root_transform.rotation * bone_data.orientation)
+        });
+}

src/xr_input/hands/mod.rs

@@ -0,0 +1,133 @@
+use bevy::{app::PluginGroupBuilder, prelude::*};
+
+use self::{emulated::HandEmulationPlugin, hand_tracking::HandTrackingPlugin};
+
+pub mod emulated;
+pub mod hand_tracking;
+pub mod common;
+
+pub struct XrHandPlugins;
+
+impl PluginGroup for XrHandPlugins {
+    fn build(self) -> PluginGroupBuilder {
+        PluginGroupBuilder::start::<Self>()
+            .add(HandTrackingPlugin)
+            .add(HandEmulationPlugin)
+            .build()
+    }
+}
+
+#[derive(Component, Debug, Clone, Copy, PartialEq)]
+pub enum BoneTrackingStatus {
+    Emulated,
+    Tracked,
+}
+
+#[derive(Component, Debug, Clone, Copy)]
+pub enum HandBone {
+    Palm,
+    Wrist,
+    ThumbMetacarpal,
+    ThumbProximal,
+    ThumbDistal,
+    ThumbTip,
+    IndexMetacarpal,
+    IndexProximal,
+    IndexIntermediate,
+    IndexDistal,
+    IndexTip,
+    MiddleMetacarpal,
+    MiddleProximal,
+    MiddleIntermediate,
+    MiddleDistal,
+    MiddleTip,
+    RingMetacarpal,
+    RingProximal,
+    RingIntermediate,
+    RingDistal,
+    RingTip,
+    LittleMetacarpal,
+    LittleProximal,
+    LittleIntermediate,
+    LittleDistal,
+    LittleTip,
+}
+impl HandBone {
+    pub fn is_finger(&self) -> bool {
+        match &self {
+            HandBone::Wrist => false,
+            HandBone::Palm => false,
+            _ => true,
+        }
+    }
+    pub fn is_metacarpal(&self) -> bool {
+        match &self {
+            HandBone::ThumbMetacarpal => true,
+            HandBone::IndexMetacarpal => true,
+            HandBone::MiddleMetacarpal => true,
+            HandBone::RingMetacarpal => true,
+            HandBone::LittleTip => true,
+            _ => false,
+        }
+    }
+    pub const fn get_all_bones() -> [HandBone; 26] {
+        [
+            HandBone::Palm,
+            HandBone::Wrist,
+            HandBone::ThumbMetacarpal,
+            HandBone::ThumbProximal,
+            HandBone::ThumbDistal,
+            HandBone::ThumbTip,
+            HandBone::IndexMetacarpal,
+            HandBone::IndexProximal,
+            HandBone::IndexIntermediate,
+            HandBone::IndexDistal,
+            HandBone::IndexTip,
+            HandBone::MiddleMetacarpal,
+            HandBone::MiddleProximal,
+            HandBone::MiddleIntermediate,
+            HandBone::MiddleDistal,
+            HandBone::MiddleTip,
+            HandBone::RingMetacarpal,
+            HandBone::RingProximal,
+            HandBone::RingIntermediate,
+            HandBone::RingDistal,
+            HandBone::RingTip,
+            HandBone::LittleMetacarpal,
+            HandBone::LittleProximal,
+            HandBone::LittleIntermediate,
+            HandBone::LittleDistal,
+            HandBone::LittleTip,
+        ]
+    }
+    pub fn get_index_from_bone(&self) -> usize {
+        match &self {
+            HandBone::Palm => 0,
+            HandBone::Wrist => 1,
+            HandBone::ThumbMetacarpal => 2,
+            HandBone::ThumbProximal => 3,
+            HandBone::ThumbDistal => 4,
+            HandBone::ThumbTip => 5,
+            HandBone::IndexMetacarpal => 6,
+            HandBone::IndexProximal => 7,
+            HandBone::IndexIntermediate => 8,
+            HandBone::IndexDistal => 9,
+            HandBone::IndexTip => 10,
+            HandBone::MiddleMetacarpal => 11,
+            HandBone::MiddleProximal => 12,
+            HandBone::MiddleIntermediate => 13,
+            HandBone::MiddleDistal => 14,
+            HandBone::MiddleTip => 15,
+            HandBone::RingMetacarpal => 16,
+            HandBone::RingProximal => 17,
+            HandBone::RingIntermediate => 18,
+            HandBone::RingDistal => 19,
+            HandBone::RingTip => 20,
+            HandBone::LittleMetacarpal => 21,
+            HandBone::LittleProximal => 22,
+            HandBone::LittleIntermediate => 23,
+            HandBone::LittleDistal => 24,
+            HandBone::LittleTip => 25,
+        }
+    }
+}

src/xr_input/handtracking.rs

@@ -1,84 +0,0 @@
-use std::mem::MaybeUninit;
-
-use bevy::prelude::*;
-use openxr::{HandJointLocationEXT, HandTracker, Result};
-
-use crate::{
-    input::XrInput,
-    resources::{XrFrameState, XrFrameWaiter, XrSession},
-};
-
-use super::hand::HandBone;
-
-#[derive(Resource)]
-pub struct HandTrackingTracker {
-    left_hand: HandTracker,
-    right_hand: HandTracker,
-}
-
-impl HandTrackingTracker {
-    pub fn new(session: &XrSession) -> Result<HandTrackingTracker> {
-        let left = session.create_hand_tracker(openxr::HandEXT::LEFT)?;
-        let right = session.create_hand_tracker(openxr::HandEXT::RIGHT)?;
-        Ok(HandTrackingTracker {
-            left_hand: left,
-            right_hand: right,
-        })
-    }
-    pub fn get_ref<'a>(
-        &'a self,
-        input: &'a XrInput,
-        frame_state: &'a XrFrameState,
-    ) -> HandTrackingRef<'a> {
-        HandTrackingRef {
-            tracking: self,
-            input,
-            frame_state,
-        }
-    }
-}
-
-pub struct HandTrackingRef<'a> {
-    tracking: &'a HandTrackingTracker,
-    input: &'a XrInput,
-    frame_state: &'a XrFrameState,
-}
-
-// pub type HandJoints = [(HandJointLocationEXT, HandBone); 26];
-
-impl<'a> HandTrackingRef<'a> {
-    pub fn get_left_poses(&self) -> Option<[HandJointLocationEXT;26]> {
-        self.input
-            .stage
-            .locate_hand_joints(
-                &self.tracking.left_hand,
-                self.frame_state.lock().unwrap().predicted_display_time,
-            )
-            .unwrap()
-            // .map(|joints| {
-            //     joints
-            //         .into_iter()
-            //         .zip(HandBone::get_all_bones().into_iter())
-            //         .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
-            //         .try_into()
-            //         .unwrap()
-            // })
-    }
-    pub fn get_right_poses(&self) -> Option<[HandJointLocationEXT;26]> {
-        self.input
-            .stage
-            .locate_hand_joints(
-                &self.tracking.right_hand,
-                self.frame_state.lock().unwrap().predicted_display_time,
-            )
-            .unwrap()
-            // .map(|joints| {
-            //     joints
-            //         .into_iter()
-            //         .zip(HandBone::get_all_bones().into_iter())
-            //         .collect::<Vec<(HandJointLocationEXT, HandBone)>>()
-            //         .try_into()
-            //         .unwrap()
-            // })
-    }
-}

src/xr_input/interactions.rs

@@ -1,7 +1,8 @@
 use std::f32::consts::PI;
 
+use bevy::log::info;
 use bevy::prelude::{
-    info, Color, Component, Entity, Event, EventReader, EventWriter, Gizmos, GlobalTransform, Quat,
+    Color, Component, Entity, Event, EventReader, EventWriter, Gizmos, GlobalTransform, Quat,
     Query, Transform, Vec3, With, Without,
 };
 
@@ -45,7 +46,7 @@ impl Default for XRInteractorState {
 #[derive(Component)]
 pub enum XRSelection {
     Empty,
-    Full(Entity)
+    Full(Entity),
 }
 impl Default for XRSelection {
     fn default() -> Self {

src/xr_input/mod.rs

@@ -1,27 +1,28 @@
 pub mod actions;
 pub mod controllers;
 pub mod debug_gizmos;
-pub mod hand;
 pub mod hand_poses;
-pub mod handtracking;
+pub mod hands;
 pub mod interactions;
 pub mod oculus_touch;
 pub mod prototype_locomotion;
 pub mod trackers;
 pub mod xr_camera;
 
-use crate::resources::XrSession;
+use crate::resources::{XrInstance, XrSession};
 use crate::xr_begin_frame;
 use crate::xr_input::controllers::XrControllerType;
 use crate::xr_input::oculus_touch::setup_oculus_controller;
 use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle};
 use bevy::app::{App, PostUpdate, Startup};
 use bevy::log::warn;
-use bevy::prelude::{BuildChildren, Component, IntoSystemConfigs};
+use bevy::prelude::{BuildChildren, Component, Deref, DerefMut, IntoSystemConfigs, Resource};
 use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3};
 use bevy::render::camera::CameraProjectionPlugin;
 use bevy::render::view::{update_frusta, VisibilitySystems};
 use bevy::transform::TransformSystem;
+use bevy::utils::HashMap;
+use openxr::Binding;
 
 use self::actions::{setup_oxr_actions, OpenXrActionsPlugin};
 use self::oculus_touch::{post_action_setup_oculus_controller, ActionSets};
@@ -75,6 +76,17 @@ impl Plugin for OpenXrInput {
     }
 }
 
+#[derive(Deref, DerefMut, Resource)]
+pub struct InteractionProfileBindings(pub HashMap<&'static str, Vec<Binding<'static>>>);
+
+fn setup_binding_recommendations(
+    mut commands: Commands,
+    instance: Res<XrInstance>,
+    bindings: Res<InteractionProfileBindings>,
+) {
+    commands.remove_resource::<InteractionProfileBindings>();
+}
+
 fn setup_xr_cameras(mut commands: Commands) {
     //this needs to do the whole xr tracking volume not just cameras
     //get the root?

src/xr_input/oculus_touch.rs

@@ -1,17 +1,16 @@
 use crate::input::XrInput;
 use crate::resources::{XrInstance, XrSession};
-use crate::xr_input::controllers::{Handed, Touchable};
+use crate::xr_input::controllers::Handed;
 use crate::xr_input::Hand;
 use bevy::prelude::{Commands, Res, ResMut, Resource};
 use openxr::{
-    Action, ActionSet, AnyGraphics, Binding, FrameState, Haptic, Instance, Path, Posef, Session,
+    ActionSet, AnyGraphics, FrameState, Instance, Path, Posef, Session, Space, SpaceLocation,
-    Space, SpaceLocation, SpaceVelocity,
+    SpaceVelocity,
 };
 
-use std::convert::identity;
 use std::sync::OnceLock;
 
-use super::actions::{ActionHandednes, XrActionSets, ActionType, SetupActionSets, XrBinding};
+use super::actions::{ActionHandednes, ActionType, SetupActionSets, XrActionSets, XrBinding};
 
 pub fn post_action_setup_oculus_controller(
     action_sets: Res<XrActionSets>,
@@ -30,34 +29,18 @@ pub fn post_action_setup_oculus_controller(
         .unwrap();
     controller.grip_space = Some(Handed {
         left: grip_action
-            .create_space(
+            .create_space(s.clone(), left_path, Posef::IDENTITY)
-                s.clone(),
-                left_path,
-                Posef::IDENTITY,
-            )
             .unwrap(),
         right: grip_action
-            .create_space(
+            .create_space(s.clone(), right_path, Posef::IDENTITY)
-                s.clone(),
-                right_path,
-                Posef::IDENTITY,
-            )
             .unwrap(),
     });
     controller.aim_space = Some(Handed {
         left: aim_action
-            .create_space(
+            .create_space(s.clone(), left_path, Posef::IDENTITY)
-                s.clone(),
-                left_path,
-                Posef::IDENTITY,
-            )
             .unwrap(),
         right: aim_action
-            .create_space(
+            .create_space(s.clone(), right_path, Posef::IDENTITY)
-                s.clone(),
-                right_path,
-                Posef::IDENTITY,
-            )
             .unwrap(),
     })
 }
@@ -100,27 +83,51 @@ pub fn subaction_path(hand: Hand) -> Path {
 impl OculusControllerRef<'_> {
     pub fn grip_space(&self, hand: Hand) -> (SpaceLocation, SpaceVelocity) {
         match hand {
-            Hand::Left => self.oculus_controller.grip_space.as_ref().unwrap().left.relate(
+            Hand::Left => self
-                &self.xr_input.stage,
+                .oculus_controller
-                self.frame_state.predicted_display_time,
+                .grip_space
-            ),
+                .as_ref()
-            Hand::Right => self.oculus_controller.grip_space.as_ref().unwrap().right.relate(
+                .unwrap()
-                &self.xr_input.stage,
+                .left
-                self.frame_state.predicted_display_time,
+                .relate(
-            ),
+                    &self.xr_input.stage,
+                    self.frame_state.predicted_display_time,
+                ),
+            Hand::Right => self
+                .oculus_controller
+                .grip_space
+                .as_ref()
+                .unwrap()
+                .right
+                .relate(
+                    &self.xr_input.stage,
+                    self.frame_state.predicted_display_time,
+                ),
         }
         .unwrap()
     }
     pub fn aim_space(&self, hand: Hand) -> (SpaceLocation, SpaceVelocity) {
         match hand {
-            Hand::Left => self.oculus_controller.aim_space.as_ref().unwrap().left.relate(
+            Hand::Left => self
-                &self.xr_input.stage,
+                .oculus_controller
-                self.frame_state.predicted_display_time,
+                .aim_space
-            ),
+                .as_ref()
-            Hand::Right => self.oculus_controller.aim_space.as_ref().unwrap().right.relate(
+                .unwrap()
-                &self.xr_input.stage,
+                .left
-                self.frame_state.predicted_display_time,
+                .relate(
-            ),
+                    &self.xr_input.stage,
+                    self.frame_state.predicted_display_time,
+                ),
+            Hand::Right => self
+                .oculus_controller
+                .aim_space
+                .as_ref()
+                .unwrap()
+                .right
+                .relate(
+                    &self.xr_input.stage,
+                    self.frame_state.predicted_display_time,
+                ),
         }
         .unwrap()
     }

src/xr_input/trackers.rs

@@ -1,11 +1,11 @@
+use bevy::log::info;
 use bevy::prelude::{
-    info, Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With,
+    Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With, Without,
-    Without,
 };
 
 use crate::{
     input::XrInput,
-    resources::{XrFrameState, XrInstance, XrSession},
+    resources::{XrFrameState, XrSession},
 };
 
 use super::{actions::XrActionSets, oculus_touch::OculusController, Hand, QuatConv, Vec3Conv};