Merge remote-tracking branch 'origin/main' into quest

examples/xr.rs

@@ -1,11 +1,24 @@
+
+
 use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin};
+
 use bevy::prelude::*;
 use bevy::transform::components::Transform;
-use bevy_openxr::xr_input::debug_gizmos::OpenXrDebugRenderer;
+use bevy_openxr::input::XrInput;
+use bevy_openxr::resources::{XrFrameState, XrInstance, XrSession};
+
+use bevy_openxr::xr_input::hand::{OpenXrHandInput, HandInputDebugRenderer};
+use bevy_openxr::xr_input::interactions::{
+    draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
+    update_interactable_states, InteractionEvent, Touched, XRDirectInteractor, XRInteractable,
+    XRInteractableState, XRInteractorState, XRRayInteractor, XRSocketInteractor,
+};
+use bevy_openxr::xr_input::oculus_touch::OculusController;
 use bevy_openxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig};
 use bevy_openxr::xr_input::trackers::{
-    OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker,
+    AimPose, OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker,
 };
+use bevy_openxr::xr_input::Hand;
 use bevy_openxr::DefaultXrPlugins;
 
 fn main() {
@@ -14,13 +27,29 @@ fn main() {
     info!("Running `openxr-6dof` skill");
     App::new()
         .add_plugins(DefaultXrPlugins)
-        .add_plugins(OpenXrDebugRenderer) //new debug renderer adds gizmos to
+        //.add_plugins(OpenXrDebugRenderer) //new debug renderer adds gizmos to
         .add_plugins(LogDiagnosticsPlugin::default())
         .add_plugins(FrameTimeDiagnosticsPlugin)
         .add_systems(Startup, setup)
         .add_systems(Update, proto_locomotion)
-        .add_systems(Startup, spawn_controllers_example)
         .insert_resource(PrototypeLocomotionConfig::default())
+        .add_systems(Startup, spawn_controllers_example)
+        .add_plugins(OpenXrHandInput)
+        .add_plugins(HandInputDebugRenderer)
+        .add_systems(
+            Update,
+            draw_interaction_gizmos.after(update_interactable_states),
+        )
+        .add_systems(Update, draw_socket_gizmos.after(update_interactable_states))
+        .add_systems(Update, interactions.before(update_interactable_states))
+        .add_systems(
+            Update,
+            socket_interactions.before(update_interactable_states),
+        )
+        .add_systems(Update, prototype_interaction_input)
+        .add_systems(Update, update_interactable_states)
+        .add_systems(Update, update_grabbables.after(update_interactable_states))
+        .add_event::<InteractionEvent>()
         .run();
 }
 
@@ -43,13 +72,16 @@ fn setup(
         transform: Transform::from_xyz(0.0, 0.5, 0.0),
         ..default()
     });
-    // cube
+    // socket
-    commands.spawn(PbrBundle {
+    commands.spawn((
-        mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
+        SpatialBundle {
-        material: materials.add(Color::rgb(0.8, 0.0, 0.0).into()),
             transform: Transform::from_xyz(0.0, 0.5, 1.0),
             ..default()
-    });
+        },
+        XRInteractorState::Selecting,
+        XRSocketInteractor,
+    ));
+
     // light
     commands.spawn(PointLightBundle {
         point_light: PointLight {
@@ -62,9 +94,27 @@ fn setup(
     });
     // camera
     commands.spawn((Camera3dBundle {
-        transform: Transform::from_xyz(-2.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y),
+        transform: Transform::from_xyz(0.25, 1.25, 0.0).looking_at(
+            Vec3 {
+                x: -0.548,
+                y: -0.161,
+                z: -0.137,
+            },
+            Vec3::Y,
+        ),
         ..default()
     },));
+    //simple interactable
+    commands.spawn((
+        SpatialBundle {
+            transform: Transform::from_xyz(0.0, 1.0, 0.0),
+            ..default()
+        },
+        XRInteractable,
+        XRInteractableState::default(),
+        Grabbable,
+        Touched(false),
+    ));
 }
 
 fn spawn_controllers_example(mut commands: Commands) {
@@ -74,6 +124,9 @@ fn spawn_controllers_example(mut commands: Commands) {
         OpenXRController,
         OpenXRTracker,
         SpatialBundle::default(),
+        XRRayInteractor,
+        AimPose(Transform::default()),
+        XRInteractorState::default(),
     ));
     //right hand
     commands.spawn((
@@ -81,5 +134,89 @@ fn spawn_controllers_example(mut commands: Commands) {
         OpenXRController,
         OpenXRTracker,
         SpatialBundle::default(),
+        XRDirectInteractor,
+        XRInteractorState::default(),
     ));
 }
+
+fn prototype_interaction_input(
+    oculus_controller: Res<OculusController>,
+    frame_state: Res<XrFrameState>,
+    xr_input: Res<XrInput>,
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+    mut right_interactor_query: Query<
+        (&mut XRInteractorState),
+        (
+            With<XRDirectInteractor>,
+            With<OpenXRRightController>,
+            Without<OpenXRLeftController>,
+        ),
+    >,
+    mut left_interactor_query: Query<
+        (&mut XRInteractorState),
+        (
+            With<XRRayInteractor>,
+            With<OpenXRLeftController>,
+            Without<OpenXRRightController>,
+        ),
+    >,
+) {
+    //lock frame
+    let frame_state = *frame_state.lock().unwrap();
+    //get controller
+    let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
+    //get controller triggers
+    let left_trigger = controller.trigger(Hand::Left);
+    let right_trigger = controller.trigger(Hand::Right);
+    //get the interactors and do state stuff
+    let mut left_state = left_interactor_query.single_mut();
+    if left_trigger > 0.8 {
+        *left_state = XRInteractorState::Selecting;
+    } else {
+        *left_state = XRInteractorState::Idle;
+    }
+    let mut right_state = right_interactor_query.single_mut();
+    if right_trigger > 0.8 {
+        *right_state = XRInteractorState::Selecting;
+    } else {
+        *right_state = XRInteractorState::Idle;
+    }
+}
+
+#[derive(Component)]
+pub struct Grabbable;
+
+pub fn update_grabbables(
+    mut events: EventReader<InteractionEvent>,
+    mut grabbable_query: Query<(&mut Transform, With<Grabbable>, Without<XRDirectInteractor>)>,
+    interactor_query: Query<(&GlobalTransform, &XRInteractorState, Without<Grabbable>)>,
+) {
+    //so basically the idea is to try all the events?
+    for event in events.read() {
+        // info!("some event");
+        match grabbable_query.get_mut(event.interactable) {
+            Ok(mut grabbable_transform) => {
+                // info!("we got a grabbable");
+                //now we need the location of our interactor
+                match interactor_query.get(event.interactor) {
+                    Ok(interactor_transform) => {
+                        match interactor_transform.1 {
+                            XRInteractorState::Idle => (),
+                            XRInteractorState::Selecting => {
+                                // info!("its a direct interactor?");
+                                *grabbable_transform.0 = interactor_transform.0.compute_transform();
+                            }
+                        }
+                    }
+                    Err(_) => {
+                        // info!("not a direct interactor")
+                    }
+                }
+            }
+            Err(_) => {
+                // info!("not a grabbable?")
+            }
+        }
+    }
+}

src/xr_input/hand_poses.rs

@@ -0,0 +1,520 @@
+use bevy::prelude::{Quat, Transform, Vec3};
+use openxr::{Posef, Quaternionf, Vector3f};
+
+use super::Hand;
+
+pub fn get_simulated_open_hand_transforms(hand: Hand) -> [Transform; 26] {
+    let test_hand_bones: [Vec3; 26] = [
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.0,
+        }, //palm
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: -0.04,
+        }, //wrist
+        Vec3 {
+            x: -0.02,
+            y: 0.00,
+            z: 0.015,
+        }, //thumb
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.03,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.024,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.024,
+        },
+        Vec3 {
+            x: -0.01,
+            y: -0.015,
+            z: 0.0155,
+        }, //index
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.064,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.037,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.02,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.01,
+        },
+        Vec3 {
+            x: 0.0,
+            y: -0.02,
+            z: 0.016,
+        }, //middle
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.064,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.037,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.02,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.01,
+        },
+        Vec3 {
+            x: 0.01,
+            y: -0.015,
+            z: 0.015,
+        }, //ring
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.064,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.037,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.02,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.01,
+        },
+        Vec3 {
+            x: 0.02,
+            y: -0.01,
+            z: 0.015,
+        }, //little
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.064,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.037,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.02,
+        },
+        Vec3 {
+            x: 0.0,
+            y: 0.0,
+            z: 0.01,
+        },
+    ];
+    let result = bones_to_transforms(test_hand_bones, hand);
+    return result;
+}
+
+fn bones_to_transforms(hand_bones: [Vec3; 26], hand: Hand) -> [Transform; 26] {
+    match hand {
+        Hand::Left => {
+            let mut result_array: [Transform; 26] = [Transform::default(); 26];
+            for (place, data) in result_array.iter_mut().zip(hand_bones.iter()) {
+                *place = Transform {
+                    translation: Vec3 {
+                        x: -data.x,
+                        y: -data.y,
+                        z: -data.z,
+                    },
+                    rotation: Quat::IDENTITY,
+                    scale: Vec3::splat(1.0),
+                }
+            }
+            return result_array;
+        }
+        Hand::Right => {
+            let mut result_array: [Transform; 26] = [Transform::default(); 26];
+            for (place, data) in result_array.iter_mut().zip(hand_bones.iter()) {
+                *place = Transform {
+                    translation: Vec3 {
+                        x: data.x,
+                        y: -data.y,
+                        z: -data.z,
+                    },
+                    rotation: Quat::IDENTITY,
+                    scale: Vec3::splat(1.0),
+                }
+            }
+            return result_array;
+        }
+    }
+}
+
+pub fn get_test_hand_pose_array() -> [Posef; 26] {
+    let test_hand_pose: [Posef; 26] = [
+        Posef {
+            position: Vector3f {
+                x: 0.0,
+                y: 0.0,
+                z: 0.0,
+            },
+            orientation: Quaternionf {
+                x: -0.267,
+                y: 0.849,
+                z: 0.204,
+                w: 0.407,
+            },
+        }, //palm
+        Posef {
+            position: Vector3f {
+                x: 0.02,
+                y: -0.040,
+                z: -0.015,
+            },
+            orientation: Quaternionf {
+                x: -0.267,
+                y: 0.849,
+                z: 0.204,
+                w: 0.407,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: 0.019,
+                y: -0.037,
+                z: 0.011,
+            },
+            orientation: Quaternionf {
+                x: -0.744,
+                y: -0.530,
+                z: 0.156,
+                w: -0.376,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: 0.015,
+                y: -0.014,
+                z: 0.047,
+            },
+            orientation: Quaternionf {
+                x: -0.786,
+                y: -0.550,
+                z: 0.126,
+                w: -0.254,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: 0.004,
+                y: 0.003,
+                z: 0.068,
+            },
+            orientation: Quaternionf {
+                x: -0.729,
+                y: -0.564,
+                z: 0.027,
+                w: -0.387,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.009,
+                y: 0.011,
+                z: 0.072,
+            },
+            orientation: Quaternionf {
+                x: -0.585,
+                y: -0.548,
+                z: -0.140,
+                w: -0.582,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: 0.027,
+                y: -0.021,
+                z: 0.001,
+            },
+            orientation: Quaternionf {
+                x: -0.277,
+                y: -0.826,
+                z: 0.317,
+                w: -0.376,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.002,
+                y: 0.026,
+                z: 0.034,
+            },
+            orientation: Quaternionf {
+                x: -0.277,
+                y: -0.826,
+                z: 0.317,
+                w: -0.376,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.023,
+                y: 0.049,
+                z: 0.055,
+            },
+            orientation: Quaternionf {
+                x: -0.244,
+                y: -0.843,
+                z: 0.256,
+                w: -0.404,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.037,
+                y: 0.059,
+                z: 0.067,
+            },
+            orientation: Quaternionf {
+                x: -0.200,
+                y: -0.866,
+                z: 0.165,
+                w: -0.428,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.045,
+                y: 0.063,
+                z: 0.073,
+            },
+            orientation: Quaternionf {
+                x: -0.172,
+                y: -0.874,
+                z: 0.110,
+                w: -0.440,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: 0.021,
+                y: -0.017,
+                z: -0.007,
+            },
+            orientation: Quaternionf {
+                x: -0.185,
+                y: -0.817,
+                z: 0.370,
+                w: -0.401,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.011,
+                y: 0.029,
+                z: 0.018,
+            },
+            orientation: Quaternionf {
+                x: -0.185,
+                y: -0.817,
+                z: 0.370,
+                w: -0.401,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.034,
+                y: 0.06,
+                z: 0.033,
+            },
+            orientation: Quaternionf {
+                x: -0.175,
+                y: -0.809,
+                z: 0.371,
+                w: -0.420,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.051,
+                y: 0.072,
+                z: 0.045,
+            },
+            orientation: Quaternionf {
+                x: -0.109,
+                y: -0.856,
+                z: 0.245,
+                w: -0.443,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.06,
+                y: 0.077,
+                z: 0.051,
+            },
+            orientation: Quaternionf {
+                x: -0.075,
+                y: -0.871,
+                z: 0.180,
+                w: -0.450,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: 0.013,
+                y: -0.017,
+                z: -0.015,
+            },
+            orientation: Quaternionf {
+                x: -0.132,
+                y: -0.786,
+                z: 0.408,
+                w: -0.445,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.02,
+                y: 0.025,
+                z: 0.0,
+            },
+            orientation: Quaternionf {
+                x: -0.132,
+                y: -0.786,
+                z: 0.408,
+                w: -0.445,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.042,
+                y: 0.055,
+                z: 0.007,
+            },
+            orientation: Quaternionf {
+                x: -0.131,
+                y: -0.762,
+                z: 0.432,
+                w: -0.464,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.06,
+                y: 0.069,
+                z: 0.015,
+            },
+            orientation: Quaternionf {
+                x: -0.071,
+                y: -0.810,
+                z: 0.332,
+                w: -0.477,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.069,
+                y: 0.075,
+                z: 0.02,
+            },
+            orientation: Quaternionf {
+                x: -0.029,
+                y: -0.836,
+                z: 0.260,
+                w: -0.482,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: 0.004,
+                y: -0.022,
+                z: -0.022,
+            },
+            orientation: Quaternionf {
+                x: -0.060,
+                y: -0.749,
+                z: 0.481,
+                w: -0.452,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.028,
+                y: 0.018,
+                z: -0.015,
+            },
+            orientation: Quaternionf {
+                x: -0.060,
+                y: -0.749,
+                z: 0.481,
+                w: -0.452,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.046,
+                y: 0.042,
+                z: -0.017,
+            },
+            orientation: Quaternionf {
+                x: -0.061,
+                y: -0.684,
+                z: 0.534,
+                w: -0.493,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.059,
+                y: 0.053,
+                z: -0.015,
+            },
+            orientation: Quaternionf {
+                x: 0.002,
+                y: -0.745,
+                z: 0.444,
+                w: -0.498,
+            },
+        },
+        Posef {
+            position: Vector3f {
+                x: -0.068,
+                y: 0.059,
+                z: -0.013,
+            },
+            orientation: Quaternionf {
+                x: 0.045,
+                y: -0.780,
+                z: 0.378,
+                w: -0.496,
+            },
+        },
+    ];
+    return test_hand_pose;
+}

src/xr_input/interactions.rs

@@ -0,0 +1,370 @@
+use std::f32::consts::PI;
+
+use bevy::prelude::{
+    info, Color, Component, Entity, Event, EventReader, EventWriter, Gizmos, GlobalTransform, Quat,
+    Query, Transform, Vec3, With, Without,
+};
+
+use super::trackers::{AimPose, OpenXRTrackingRoot};
+
+#[derive(Component)]
+pub struct XRDirectInteractor;
+
+#[derive(Component)]
+pub struct XRRayInteractor;
+
+#[derive(Component)]
+pub struct XRSocketInteractor;
+
+#[derive(Component)]
+pub struct Touched(pub bool);
+
+#[derive(Component, Clone, Copy, PartialEq, PartialOrd, Debug)]
+pub enum XRInteractableState {
+    Idle,
+    Hover,
+    Select,
+}
+
+impl Default for XRInteractableState {
+    fn default() -> Self {
+        XRInteractableState::Idle
+    }
+}
+
+#[derive(Component)]
+pub enum XRInteractorState {
+    Idle,
+    Selecting,
+}
+impl Default for XRInteractorState {
+    fn default() -> Self {
+        XRInteractorState::Idle
+    }
+}
+
+#[derive(Component)]
+pub struct XRInteractable;
+
+pub fn draw_socket_gizmos(
+    mut gizmos: Gizmos,
+    interactor_query: Query<(
+        &GlobalTransform,
+        &XRInteractorState,
+        Entity,
+        &XRSocketInteractor,
+    )>,
+) {
+    for (global, state, _entity, _socket) in interactor_query.iter() {
+        let mut transform = global.compute_transform().clone();
+        transform.scale = Vec3::splat(0.1);
+        let color = match state {
+            XRInteractorState::Idle => Color::BLUE,
+            XRInteractorState::Selecting => Color::PURPLE,
+        };
+        gizmos.cuboid(transform, color)
+    }
+}
+
+pub fn draw_interaction_gizmos(
+    mut gizmos: Gizmos,
+    interactable_query: Query<
+        (&GlobalTransform, &XRInteractableState),
+        (With<XRInteractable>, Without<XRDirectInteractor>),
+    >,
+    interactor_query: Query<
+        (
+            &GlobalTransform,
+            &XRInteractorState,
+            Option<&XRDirectInteractor>,
+            Option<&XRRayInteractor>,
+            Option<&AimPose>,
+        ),
+        Without<XRInteractable>,
+    >,
+    tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
+) {
+    let root = tracking_root_query.get_single().unwrap().0;
+    for (global_transform, interactable_state) in interactable_query.iter() {
+        let transform = global_transform.compute_transform();
+        let color = match interactable_state {
+            XRInteractableState::Idle => Color::RED,
+            XRInteractableState::Hover => Color::YELLOW,
+            XRInteractableState::Select => Color::GREEN,
+        };
+        gizmos.sphere(transform.translation, transform.rotation, 0.1, color);
+    }
+
+    for (interactor_global_transform, interactor_state, direct, ray, aim) in interactor_query.iter()
+    {
+        let transform = interactor_global_transform.compute_transform();
+        match direct {
+            Some(_) => {
+                let mut local = transform.clone();
+                local.scale = Vec3::splat(0.1);
+                let quat = Quat::from_euler(
+                    bevy::prelude::EulerRot::XYZ,
+                    45.0 * (PI / 180.0),
+                    0.0,
+                    45.0 * (PI / 180.0),
+                );
+                local.rotation = quat;
+                let color = match interactor_state {
+                    XRInteractorState::Idle => Color::BLUE,
+                    XRInteractorState::Selecting => Color::PURPLE,
+                };
+                gizmos.cuboid(local, color);
+            }
+            None => (),
+        }
+        match ray {
+            Some(_) => match aim {
+                Some(aim) => {
+                    let color = match interactor_state {
+                        XRInteractorState::Idle => Color::BLUE,
+                        XRInteractorState::Selecting => Color::PURPLE,
+                    };
+                    gizmos.ray(
+                        root.translation + root.rotation.mul_vec3(aim.0.translation),
+                        root.rotation.mul_vec3(aim.0.forward()),
+                        color,
+                    );
+                }
+                None => todo!(),
+            },
+            None => (),
+        }
+    }
+}
+
+#[derive(Event)]
+pub struct InteractionEvent {
+    pub interactor: Entity,
+    pub interactable: Entity,
+    pub interactable_state: XRInteractableState,
+}
+
+pub fn socket_interactions(
+    interactable_query: Query<
+        (&GlobalTransform, &mut XRInteractableState, Entity),
+        (With<XRInteractable>, Without<XRSocketInteractor>),
+    >,
+    interactor_query: Query<
+        (
+            &GlobalTransform,
+            &XRInteractorState,
+            Entity,
+            &XRSocketInteractor,
+        ),
+        Without<XRInteractable>,
+    >,
+    mut writer: EventWriter<InteractionEvent>,
+) {
+    for interactable in interactable_query.iter() {
+        //for the interactables
+        for socket in interactor_query.iter() {
+            let interactor_global_transform = socket.0;
+            let xr_interactable_global_transform = interactable.0;
+            let interactor_state = socket.1;
+            //check for sphere overlaps
+            let size = 0.1;
+            if interactor_global_transform
+                .compute_transform()
+                .translation
+                .distance_squared(
+                    xr_interactable_global_transform
+                        .compute_transform()
+                        .translation,
+                )
+                < (size * size) * 2.0
+            {
+                //check for selections first
+                match interactor_state {
+                    XRInteractorState::Idle => {
+                        let event = InteractionEvent {
+                            interactor: socket.2,
+                            interactable: interactable.2,
+                            interactable_state: XRInteractableState::Hover,
+                        };
+                        writer.send(event);
+                    }
+                    XRInteractorState::Selecting => {
+                        let event = InteractionEvent {
+                            interactor: socket.2,
+                            interactable: interactable.2,
+                            interactable_state: XRInteractableState::Select,
+                        };
+                        writer.send(event);
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub fn interactions(
+    interactable_query: Query<
+        (&GlobalTransform, Entity),
+        (With<XRInteractable>, Without<XRDirectInteractor>),
+    >,
+    interactor_query: Query<
+        (
+            &GlobalTransform,
+            &XRInteractorState,
+            Entity,
+            Option<&XRDirectInteractor>,
+            Option<&XRRayInteractor>,
+            Option<&AimPose>,
+        ),
+        Without<XRInteractable>,
+    >,
+    tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
+    mut writer: EventWriter<InteractionEvent>,
+) {
+    for (xr_interactable_global_transform, interactable_entity) in interactable_query.iter() {
+        for (interactor_global_transform, interactor_state, interactor_entity, direct, ray, aim) in
+            interactor_query.iter()
+        {
+            match direct {
+                Some(_) => {
+                    //check for sphere overlaps
+                    let size = 0.1;
+                    if interactor_global_transform
+                        .compute_transform()
+                        .translation
+                        .distance_squared(
+                            xr_interactable_global_transform
+                                .compute_transform()
+                                .translation,
+                        )
+                        < (size * size) * 2.0
+                    {
+                        //check for selections first
+                        match interactor_state {
+                            XRInteractorState::Idle => {
+                                let event = InteractionEvent {
+                                    interactor: interactor_entity,
+                                    interactable: interactable_entity,
+                                    interactable_state: XRInteractableState::Hover,
+                                };
+                                writer.send(event);
+                            }
+                            XRInteractorState::Selecting => {
+                                let event = InteractionEvent {
+                                    interactor: interactor_entity,
+                                    interactable: interactable_entity,
+                                    interactable_state: XRInteractableState::Select,
+                                };
+                                writer.send(event);
+                            }
+                        }
+                    }
+                }
+                None => (),
+            }
+            match ray {
+                Some(_) => {
+                    //check for ray-sphere intersection
+                    let sphere_transform = xr_interactable_global_transform.compute_transform();
+                    let center = sphere_transform.translation;
+                    let radius: f32 = 0.1;
+                    //I hate this but the aim pose needs the root for now
+                    let root = tracking_root_query.get_single().unwrap().0;
+                    match aim {
+                        Some(aim) => {
+                            let ray_origin =
+                                root.translation + root.rotation.mul_vec3(aim.0.translation);
+                            let ray_dir = root.rotation.mul_vec3(aim.0.forward());
+
+                            if ray_sphere_intersection(
+                                center,
+                                radius,
+                                ray_origin,
+                                ray_dir.normalize_or_zero(),
+                            ) {
+                                //check for selections first
+                                match interactor_state {
+                                    XRInteractorState::Idle => {
+                                        let event = InteractionEvent {
+                                            interactor: interactor_entity,
+                                            interactable: interactable_entity,
+                                            interactable_state: XRInteractableState::Hover,
+                                        };
+                                        writer.send(event);
+                                    }
+                                    XRInteractorState::Selecting => {
+                                        let event = InteractionEvent {
+                                            interactor: interactor_entity,
+                                            interactable: interactable_entity,
+                                            interactable_state: XRInteractableState::Select,
+                                        };
+                                        writer.send(event);
+                                    }
+                                }
+                            }
+                        }
+                        None => info!("no aim pose"),
+                    }
+                }
+                None => (),
+            }
+        }
+    }
+}
+
+pub fn update_interactable_states(
+    mut events: EventReader<InteractionEvent>,
+    mut interactable_query: Query<
+        (Entity, &mut XRInteractableState, &mut Touched),
+        With<XRInteractable>,
+    >,
+) {
+    //i very much dislike this
+    for (_entity, _state, mut touched) in interactable_query.iter_mut() {
+        *touched = Touched(false);
+    }
+    for event in events.read() {
+        //lets change the state
+        match interactable_query.get_mut(event.interactable) {
+            Ok((_entity, mut entity_state, mut touched)) => {
+                //since we have an event we were touched this frame, i hate this name
+                *touched = Touched(true);
+                if event.interactable_state > *entity_state {
+                    // info!(
+                    //     "event.state: {:?}, interactable.state: {:?}",
+                    //     event.interactable_state, entity_state
+                    // );
+                    // info!("event has a higher state");
+                }
+                *entity_state = event.interactable_state;
+            }
+            Err(_) => {}
+        }
+    }
+    //lets go through all the untouched interactables and set them to idle
+    for (_entity, mut state, touched) in interactable_query.iter_mut() {
+        if !touched.0 {
+            *state = XRInteractableState::Idle;
+        }
+    }
+}
+
+fn ray_sphere_intersection(center: Vec3, radius: f32, ray_origin: Vec3, ray_dir: Vec3) -> bool {
+    let l = center - ray_origin;
+    let adj = l.dot(ray_dir);
+    let d2 = l.dot(l) - (adj * adj);
+    let radius2 = radius * radius;
+    if d2 > radius2 {
+        return false;
+    }
+    let thc = (radius2 - d2).sqrt();
+    let t0 = adj - thc;
+    let t1 = adj + thc;
+
+    if t0 < 0.0 && t1 < 0.0 {
+        return false;
+    }
+
+    // let distance = if t0 < t1 { t0 } else { t1 };
+    return true;
+}

src/xr_input/mod.rs

@@ -1,9 +1,12 @@
 pub mod controllers;
 pub mod debug_gizmos;
+pub mod interactions;
 pub mod oculus_touch;
 pub mod prototype_locomotion;
 pub mod trackers;
 pub mod xr_camera;
+pub mod hand_poses;
+pub mod hand;
 
 use crate::resources::XrSession;
 use crate::xr_begin_frame;
@@ -12,7 +15,7 @@ use crate::xr_input::oculus_touch::{setup_oculus_controller, ActionSets};
 use crate::xr_input::xr_camera::{xr_camera_head_sync, Eye, XRProjection, XrCameraBundle};
 use bevy::app::{App, PostUpdate, Startup};
 use bevy::log::warn;
-use bevy::prelude::{BuildChildren, IntoSystemConfigs};
+use bevy::prelude::{BuildChildren, IntoSystemConfigs, Component};
 use bevy::prelude::{Commands, Plugin, PreUpdate, Quat, Res, SpatialBundle, Update, Vec3};
 use bevy::render::camera::CameraProjectionPlugin;
 use bevy::render::view::{update_frusta, VisibilitySystems};
@@ -27,7 +30,7 @@ use self::trackers::{
 pub struct OpenXrInput {
     pub controller_type: XrControllerType,
 }
-#[derive(Clone, Copy, Debug, Ord, PartialOrd, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Ord, PartialOrd, Eq, PartialEq, Component)]
 pub enum Hand {
     Left,
     Right,

src/xr_input/prototype_locomotion.rs

@@ -43,7 +43,7 @@ pub struct PrototypeLocomotionConfig {
 impl Default for PrototypeLocomotionConfig {
     fn default() -> Self {
         Self {
-            locomotion_type: LocomotionType::Hand,
+            locomotion_type: LocomotionType::Head,
             locomotion_speed: 1.0,
             rotation_type: RotationType::Smooth,
             snap_angle: 45.0 * (PI / 180.0),

src/xr_input/trackers.rs

@@ -1,8 +1,14 @@
-use bevy::prelude::{Added, BuildChildren, Commands, Entity, Query, With, Res, Transform, Without, Component, info};
+use bevy::prelude::{
+    info, Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With,
+    Without,
+};
 
-use crate::{resources::{XrFrameState, XrInstance, XrSession}, input::XrInput};
+use crate::{
+    input::XrInput,
+    resources::{XrFrameState, XrInstance, XrSession},
+};
 
-use super::{oculus_touch::OculusController, Hand, Vec3Conv, QuatConv};
+use super::{oculus_touch::OculusController, Hand, QuatConv, Vec3Conv};
 
 #[derive(Component)]
 pub struct OpenXRTrackingRoot;
@@ -20,6 +26,8 @@ pub struct OpenXRLeftController;
 pub struct OpenXRRightController;
 #[derive(Component)]
 pub struct OpenXRController;
+#[derive(Component)]
+pub struct AimPose(pub Transform);
 
 pub fn adopt_open_xr_trackers(
     query: Query<Entity, Added<OpenXRTracker>>,
@@ -43,11 +51,13 @@ pub fn update_open_xr_controllers(
     oculus_controller: Res<OculusController>,
     mut left_controller_query: Query<(
         &mut Transform,
+        Option<&mut AimPose>,
         With<OpenXRLeftController>,
         Without<OpenXRRightController>,
     )>,
     mut right_controller_query: Query<(
         &mut Transform,
+        Option<&mut AimPose>,
         With<OpenXRRightController>,
         Without<OpenXRLeftController>,
     )>,
@@ -61,8 +71,20 @@ pub fn update_open_xr_controllers(
     //get controller
     let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
     //get left controller
-    let left = controller.grip_space(Hand::Left);
+    let left_grip_space = controller.grip_space(Hand::Left);
-    let left_postion = left.0.pose.position.to_vec3();
+    let left_aim_space = controller.aim_space(Hand::Left);
+    let left_postion = left_grip_space.0.pose.position.to_vec3();
+    let left_aim_pose = left_controller_query.get_single_mut().unwrap().1;
+    match left_aim_pose {
+        Some(mut pose) => {
+            *pose = AimPose(Transform {
+                translation: left_aim_space.0.pose.position.to_vec3(),
+                rotation: left_aim_space.0.pose.orientation.to_quat(),
+                scale: Vec3::splat(1.0),
+            });
+        }
+        None => (),
+    }
 
     left_controller_query
         .get_single_mut()
@@ -70,10 +92,24 @@ pub fn update_open_xr_controllers(
         .0
         .translation = left_postion;
 
-    left_controller_query.get_single_mut().unwrap().0.rotation = left.0.pose.orientation.to_quat();
+    left_controller_query.get_single_mut().unwrap().0.rotation =
+        left_grip_space.0.pose.orientation.to_quat();
     //get right controller
-    let right = controller.grip_space(Hand::Right);
+    let right_grip_space = controller.grip_space(Hand::Right);
-    let right_postion = right.0.pose.position.to_vec3();
+    let right_aim_space = controller.aim_space(Hand::Right);
+    let right_postion = right_grip_space.0.pose.position.to_vec3();
+
+    let right_aim_pose = right_controller_query.get_single_mut().unwrap().1;
+    match right_aim_pose {
+        Some(mut pose) => {
+            *pose = AimPose(Transform {
+                translation: right_aim_space.0.pose.position.to_vec3(),
+                rotation: right_aim_space.0.pose.orientation.to_quat(),
+                scale: Vec3::splat(1.0),
+            });
+        }
+        None => (),
+    }
 
     right_controller_query
         .get_single_mut()
@@ -82,6 +118,5 @@ pub fn update_open_xr_controllers(
         .translation = right_postion;
 
     right_controller_query.get_single_mut().unwrap().0.rotation =
-        right.0.pose.orientation.to_quat();
+        right_grip_space.0.pose.orientation.to_quat();
 }
-