Merge branch 'main' into hands

examples/xr.rs

@@ -4,12 +4,21 @@ use bevy::diagnostic::{FrameTimeDiagnosticsPlugin, LogDiagnosticsPlugin};
 
 use bevy::prelude::*;
 use bevy::transform::components::Transform;
+use bevy_openxr::input::XrInput;
+use bevy_openxr::resources::{XrFrameState, XrInstance, XrSession};
 
 use bevy_openxr::xr_input::hand::{OpenXrHandInput, HandInputDebugRenderer};
+use bevy_openxr::xr_input::interactions::{
+    draw_interaction_gizmos, draw_socket_gizmos, interactions, socket_interactions,
+    update_interactable_states, InteractionEvent, Touched, XRDirectInteractor, XRInteractable,
+    XRInteractableState, XRInteractorState, XRRayInteractor, XRSocketInteractor,
+};
+use bevy_openxr::xr_input::oculus_touch::OculusController;
 use bevy_openxr::xr_input::prototype_locomotion::{proto_locomotion, PrototypeLocomotionConfig};
 use bevy_openxr::xr_input::trackers::{
-    OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker,
+    AimPose, OpenXRController, OpenXRLeftController, OpenXRRightController, OpenXRTracker,
 };
+use bevy_openxr::xr_input::Hand;
 use bevy_openxr::DefaultXrPlugins;
 
 fn main() {
@@ -27,6 +36,20 @@ fn main() {
         .add_systems(Startup, spawn_controllers_example)
         .add_plugins(OpenXrHandInput)
         .add_plugins(HandInputDebugRenderer)
+        .add_systems(
+            Update,
+            draw_interaction_gizmos.after(update_interactable_states),
+        )
+        .add_systems(Update, draw_socket_gizmos.after(update_interactable_states))
+        .add_systems(Update, interactions.before(update_interactable_states))
+        .add_systems(
+            Update,
+            socket_interactions.before(update_interactable_states),
+        )
+        .add_systems(Update, prototype_interaction_input)
+        .add_systems(Update, update_interactable_states)
+        .add_systems(Update, update_grabbables.after(update_interactable_states))
+        .add_event::<InteractionEvent>()
         .run();
 }
 
@@ -49,13 +72,16 @@ fn setup(
         transform: Transform::from_xyz(0.0, 0.5, 0.0),
         ..default()
     });
-    // cube
+    // socket
-    commands.spawn(PbrBundle {
+    commands.spawn((
-        mesh: meshes.add(Mesh::from(shape::Cube { size: 0.1 })),
+        SpatialBundle {
-        material: materials.add(Color::rgb(0.8, 0.0, 0.0).into()),
             transform: Transform::from_xyz(0.0, 0.5, 1.0),
             ..default()
-    });
+        },
+        XRInteractorState::Selecting,
+        XRSocketInteractor,
+    ));
+
     // light
     commands.spawn(PointLightBundle {
         point_light: PointLight {
@@ -78,6 +104,17 @@ fn setup(
         ),
         ..default()
     },));
+    //simple interactable
+    commands.spawn((
+        SpatialBundle {
+            transform: Transform::from_xyz(0.0, 1.0, 0.0),
+            ..default()
+        },
+        XRInteractable,
+        XRInteractableState::default(),
+        Grabbable,
+        Touched(false),
+    ));
 }
 
 fn spawn_controllers_example(mut commands: Commands) {
@@ -87,6 +124,9 @@ fn spawn_controllers_example(mut commands: Commands) {
         OpenXRController,
         OpenXRTracker,
         SpatialBundle::default(),
+        XRRayInteractor,
+        AimPose(Transform::default()),
+        XRInteractorState::default(),
     ));
     //right hand
     commands.spawn((
@@ -94,5 +134,89 @@ fn spawn_controllers_example(mut commands: Commands) {
         OpenXRController,
         OpenXRTracker,
         SpatialBundle::default(),
+        XRDirectInteractor,
+        XRInteractorState::default(),
     ));
 }
+
+fn prototype_interaction_input(
+    oculus_controller: Res<OculusController>,
+    frame_state: Res<XrFrameState>,
+    xr_input: Res<XrInput>,
+    instance: Res<XrInstance>,
+    session: Res<XrSession>,
+    mut right_interactor_query: Query<
+        (&mut XRInteractorState),
+        (
+            With<XRDirectInteractor>,
+            With<OpenXRRightController>,
+            Without<OpenXRLeftController>,
+        ),
+    >,
+    mut left_interactor_query: Query<
+        (&mut XRInteractorState),
+        (
+            With<XRRayInteractor>,
+            With<OpenXRLeftController>,
+            Without<OpenXRRightController>,
+        ),
+    >,
+) {
+    //lock frame
+    let frame_state = *frame_state.lock().unwrap();
+    //get controller
+    let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
+    //get controller triggers
+    let left_trigger = controller.trigger(Hand::Left);
+    let right_trigger = controller.trigger(Hand::Right);
+    //get the interactors and do state stuff
+    let mut left_state = left_interactor_query.single_mut();
+    if left_trigger > 0.8 {
+        *left_state = XRInteractorState::Selecting;
+    } else {
+        *left_state = XRInteractorState::Idle;
+    }
+    let mut right_state = right_interactor_query.single_mut();
+    if right_trigger > 0.8 {
+        *right_state = XRInteractorState::Selecting;
+    } else {
+        *right_state = XRInteractorState::Idle;
+    }
+}
+
+#[derive(Component)]
+pub struct Grabbable;
+
+pub fn update_grabbables(
+    mut events: EventReader<InteractionEvent>,
+    mut grabbable_query: Query<(&mut Transform, With<Grabbable>, Without<XRDirectInteractor>)>,
+    interactor_query: Query<(&GlobalTransform, &XRInteractorState, Without<Grabbable>)>,
+) {
+    //so basically the idea is to try all the events?
+    for event in events.read() {
+        // info!("some event");
+        match grabbable_query.get_mut(event.interactable) {
+            Ok(mut grabbable_transform) => {
+                // info!("we got a grabbable");
+                //now we need the location of our interactor
+                match interactor_query.get(event.interactor) {
+                    Ok(interactor_transform) => {
+                        match interactor_transform.1 {
+                            XRInteractorState::Idle => (),
+                            XRInteractorState::Selecting => {
+                                // info!("its a direct interactor?");
+                                *grabbable_transform.0 = interactor_transform.0.compute_transform();
+                            }
+                        }
+                    }
+                    Err(_) => {
+                        // info!("not a direct interactor")
+                    }
+                }
+            }
+            Err(_) => {
+                // info!("not a grabbable?")
+            }
+        }
+    }
+}

src/xr_input/interactions.rs

@@ -0,0 +1,370 @@
+use std::f32::consts::PI;
+
+use bevy::prelude::{
+    info, Color, Component, Entity, Event, EventReader, EventWriter, Gizmos, GlobalTransform, Quat,
+    Query, Transform, Vec3, With, Without,
+};
+
+use super::trackers::{AimPose, OpenXRTrackingRoot};
+
+#[derive(Component)]
+pub struct XRDirectInteractor;
+
+#[derive(Component)]
+pub struct XRRayInteractor;
+
+#[derive(Component)]
+pub struct XRSocketInteractor;
+
+#[derive(Component)]
+pub struct Touched(pub bool);
+
+#[derive(Component, Clone, Copy, PartialEq, PartialOrd, Debug)]
+pub enum XRInteractableState {
+    Idle,
+    Hover,
+    Select,
+}
+
+impl Default for XRInteractableState {
+    fn default() -> Self {
+        XRInteractableState::Idle
+    }
+}
+
+#[derive(Component)]
+pub enum XRInteractorState {
+    Idle,
+    Selecting,
+}
+impl Default for XRInteractorState {
+    fn default() -> Self {
+        XRInteractorState::Idle
+    }
+}
+
+#[derive(Component)]
+pub struct XRInteractable;
+
+pub fn draw_socket_gizmos(
+    mut gizmos: Gizmos,
+    interactor_query: Query<(
+        &GlobalTransform,
+        &XRInteractorState,
+        Entity,
+        &XRSocketInteractor,
+    )>,
+) {
+    for (global, state, _entity, _socket) in interactor_query.iter() {
+        let mut transform = global.compute_transform().clone();
+        transform.scale = Vec3::splat(0.1);
+        let color = match state {
+            XRInteractorState::Idle => Color::BLUE,
+            XRInteractorState::Selecting => Color::PURPLE,
+        };
+        gizmos.cuboid(transform, color)
+    }
+}
+
+pub fn draw_interaction_gizmos(
+    mut gizmos: Gizmos,
+    interactable_query: Query<
+        (&GlobalTransform, &XRInteractableState),
+        (With<XRInteractable>, Without<XRDirectInteractor>),
+    >,
+    interactor_query: Query<
+        (
+            &GlobalTransform,
+            &XRInteractorState,
+            Option<&XRDirectInteractor>,
+            Option<&XRRayInteractor>,
+            Option<&AimPose>,
+        ),
+        Without<XRInteractable>,
+    >,
+    tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
+) {
+    let root = tracking_root_query.get_single().unwrap().0;
+    for (global_transform, interactable_state) in interactable_query.iter() {
+        let transform = global_transform.compute_transform();
+        let color = match interactable_state {
+            XRInteractableState::Idle => Color::RED,
+            XRInteractableState::Hover => Color::YELLOW,
+            XRInteractableState::Select => Color::GREEN,
+        };
+        gizmos.sphere(transform.translation, transform.rotation, 0.1, color);
+    }
+
+    for (interactor_global_transform, interactor_state, direct, ray, aim) in interactor_query.iter()
+    {
+        let transform = interactor_global_transform.compute_transform();
+        match direct {
+            Some(_) => {
+                let mut local = transform.clone();
+                local.scale = Vec3::splat(0.1);
+                let quat = Quat::from_euler(
+                    bevy::prelude::EulerRot::XYZ,
+                    45.0 * (PI / 180.0),
+                    0.0,
+                    45.0 * (PI / 180.0),
+                );
+                local.rotation = quat;
+                let color = match interactor_state {
+                    XRInteractorState::Idle => Color::BLUE,
+                    XRInteractorState::Selecting => Color::PURPLE,
+                };
+                gizmos.cuboid(local, color);
+            }
+            None => (),
+        }
+        match ray {
+            Some(_) => match aim {
+                Some(aim) => {
+                    let color = match interactor_state {
+                        XRInteractorState::Idle => Color::BLUE,
+                        XRInteractorState::Selecting => Color::PURPLE,
+                    };
+                    gizmos.ray(
+                        root.translation + root.rotation.mul_vec3(aim.0.translation),
+                        root.rotation.mul_vec3(aim.0.forward()),
+                        color,
+                    );
+                }
+                None => todo!(),
+            },
+            None => (),
+        }
+    }
+}
+
+#[derive(Event)]
+pub struct InteractionEvent {
+    pub interactor: Entity,
+    pub interactable: Entity,
+    pub interactable_state: XRInteractableState,
+}
+
+pub fn socket_interactions(
+    interactable_query: Query<
+        (&GlobalTransform, &mut XRInteractableState, Entity),
+        (With<XRInteractable>, Without<XRSocketInteractor>),
+    >,
+    interactor_query: Query<
+        (
+            &GlobalTransform,
+            &XRInteractorState,
+            Entity,
+            &XRSocketInteractor,
+        ),
+        Without<XRInteractable>,
+    >,
+    mut writer: EventWriter<InteractionEvent>,
+) {
+    for interactable in interactable_query.iter() {
+        //for the interactables
+        for socket in interactor_query.iter() {
+            let interactor_global_transform = socket.0;
+            let xr_interactable_global_transform = interactable.0;
+            let interactor_state = socket.1;
+            //check for sphere overlaps
+            let size = 0.1;
+            if interactor_global_transform
+                .compute_transform()
+                .translation
+                .distance_squared(
+                    xr_interactable_global_transform
+                        .compute_transform()
+                        .translation,
+                )
+                < (size * size) * 2.0
+            {
+                //check for selections first
+                match interactor_state {
+                    XRInteractorState::Idle => {
+                        let event = InteractionEvent {
+                            interactor: socket.2,
+                            interactable: interactable.2,
+                            interactable_state: XRInteractableState::Hover,
+                        };
+                        writer.send(event);
+                    }
+                    XRInteractorState::Selecting => {
+                        let event = InteractionEvent {
+                            interactor: socket.2,
+                            interactable: interactable.2,
+                            interactable_state: XRInteractableState::Select,
+                        };
+                        writer.send(event);
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub fn interactions(
+    interactable_query: Query<
+        (&GlobalTransform, Entity),
+        (With<XRInteractable>, Without<XRDirectInteractor>),
+    >,
+    interactor_query: Query<
+        (
+            &GlobalTransform,
+            &XRInteractorState,
+            Entity,
+            Option<&XRDirectInteractor>,
+            Option<&XRRayInteractor>,
+            Option<&AimPose>,
+        ),
+        Without<XRInteractable>,
+    >,
+    tracking_root_query: Query<(&mut Transform, With<OpenXRTrackingRoot>)>,
+    mut writer: EventWriter<InteractionEvent>,
+) {
+    for (xr_interactable_global_transform, interactable_entity) in interactable_query.iter() {
+        for (interactor_global_transform, interactor_state, interactor_entity, direct, ray, aim) in
+            interactor_query.iter()
+        {
+            match direct {
+                Some(_) => {
+                    //check for sphere overlaps
+                    let size = 0.1;
+                    if interactor_global_transform
+                        .compute_transform()
+                        .translation
+                        .distance_squared(
+                            xr_interactable_global_transform
+                                .compute_transform()
+                                .translation,
+                        )
+                        < (size * size) * 2.0
+                    {
+                        //check for selections first
+                        match interactor_state {
+                            XRInteractorState::Idle => {
+                                let event = InteractionEvent {
+                                    interactor: interactor_entity,
+                                    interactable: interactable_entity,
+                                    interactable_state: XRInteractableState::Hover,
+                                };
+                                writer.send(event);
+                            }
+                            XRInteractorState::Selecting => {
+                                let event = InteractionEvent {
+                                    interactor: interactor_entity,
+                                    interactable: interactable_entity,
+                                    interactable_state: XRInteractableState::Select,
+                                };
+                                writer.send(event);
+                            }
+                        }
+                    }
+                }
+                None => (),
+            }
+            match ray {
+                Some(_) => {
+                    //check for ray-sphere intersection
+                    let sphere_transform = xr_interactable_global_transform.compute_transform();
+                    let center = sphere_transform.translation;
+                    let radius: f32 = 0.1;
+                    //I hate this but the aim pose needs the root for now
+                    let root = tracking_root_query.get_single().unwrap().0;
+                    match aim {
+                        Some(aim) => {
+                            let ray_origin =
+                                root.translation + root.rotation.mul_vec3(aim.0.translation);
+                            let ray_dir = root.rotation.mul_vec3(aim.0.forward());
+
+                            if ray_sphere_intersection(
+                                center,
+                                radius,
+                                ray_origin,
+                                ray_dir.normalize_or_zero(),
+                            ) {
+                                //check for selections first
+                                match interactor_state {
+                                    XRInteractorState::Idle => {
+                                        let event = InteractionEvent {
+                                            interactor: interactor_entity,
+                                            interactable: interactable_entity,
+                                            interactable_state: XRInteractableState::Hover,
+                                        };
+                                        writer.send(event);
+                                    }
+                                    XRInteractorState::Selecting => {
+                                        let event = InteractionEvent {
+                                            interactor: interactor_entity,
+                                            interactable: interactable_entity,
+                                            interactable_state: XRInteractableState::Select,
+                                        };
+                                        writer.send(event);
+                                    }
+                                }
+                            }
+                        }
+                        None => info!("no aim pose"),
+                    }
+                }
+                None => (),
+            }
+        }
+    }
+}
+
+pub fn update_interactable_states(
+    mut events: EventReader<InteractionEvent>,
+    mut interactable_query: Query<
+        (Entity, &mut XRInteractableState, &mut Touched),
+        With<XRInteractable>,
+    >,
+) {
+    //i very much dislike this
+    for (_entity, _state, mut touched) in interactable_query.iter_mut() {
+        *touched = Touched(false);
+    }
+    for event in events.read() {
+        //lets change the state
+        match interactable_query.get_mut(event.interactable) {
+            Ok((_entity, mut entity_state, mut touched)) => {
+                //since we have an event we were touched this frame, i hate this name
+                *touched = Touched(true);
+                if event.interactable_state > *entity_state {
+                    // info!(
+                    //     "event.state: {:?}, interactable.state: {:?}",
+                    //     event.interactable_state, entity_state
+                    // );
+                    // info!("event has a higher state");
+                }
+                *entity_state = event.interactable_state;
+            }
+            Err(_) => {}
+        }
+    }
+    //lets go through all the untouched interactables and set them to idle
+    for (_entity, mut state, touched) in interactable_query.iter_mut() {
+        if !touched.0 {
+            *state = XRInteractableState::Idle;
+        }
+    }
+}
+
+fn ray_sphere_intersection(center: Vec3, radius: f32, ray_origin: Vec3, ray_dir: Vec3) -> bool {
+    let l = center - ray_origin;
+    let adj = l.dot(ray_dir);
+    let d2 = l.dot(l) - (adj * adj);
+    let radius2 = radius * radius;
+    if d2 > radius2 {
+        return false;
+    }
+    let thc = (radius2 - d2).sqrt();
+    let t0 = adj - thc;
+    let t1 = adj + thc;
+
+    if t0 < 0.0 && t1 < 0.0 {
+        return false;
+    }
+
+    // let distance = if t0 < t1 { t0 } else { t1 };
+    return true;
+}

src/xr_input/mod.rs

@@ -1,5 +1,6 @@
 pub mod controllers;
 pub mod debug_gizmos;
+pub mod interactions;
 pub mod oculus_touch;
 pub mod prototype_locomotion;
 pub mod trackers;

src/xr_input/prototype_locomotion.rs

@@ -43,7 +43,7 @@ pub struct PrototypeLocomotionConfig {
 impl Default for PrototypeLocomotionConfig {
     fn default() -> Self {
         Self {
-            locomotion_type: LocomotionType::Hand,
+            locomotion_type: LocomotionType::Head,
             locomotion_speed: 1.0,
             rotation_type: RotationType::Smooth,
             snap_angle: 45.0 * (PI / 180.0),

src/xr_input/trackers.rs

@@ -1,8 +1,14 @@
-use bevy::prelude::{Added, BuildChildren, Commands, Entity, Query, With, Res, Transform, Without, Component, info};
+use bevy::prelude::{
+    info, Added, BuildChildren, Commands, Component, Entity, Query, Res, Transform, Vec3, With,
+    Without,
+};
 
-use crate::{resources::{XrFrameState, XrInstance, XrSession}, input::XrInput};
+use crate::{
+    input::XrInput,
+    resources::{XrFrameState, XrInstance, XrSession},
+};
 
-use super::{oculus_touch::OculusController, Hand, Vec3Conv, QuatConv};
+use super::{oculus_touch::OculusController, Hand, QuatConv, Vec3Conv};
 
 #[derive(Component)]
 pub struct OpenXRTrackingRoot;
@@ -20,6 +26,8 @@ pub struct OpenXRLeftController;
 pub struct OpenXRRightController;
 #[derive(Component)]
 pub struct OpenXRController;
+#[derive(Component)]
+pub struct AimPose(pub Transform);
 
 pub fn adopt_open_xr_trackers(
     query: Query<Entity, Added<OpenXRTracker>>,
@@ -43,11 +51,13 @@ pub fn update_open_xr_controllers(
     oculus_controller: Res<OculusController>,
     mut left_controller_query: Query<(
         &mut Transform,
+        Option<&mut AimPose>,
         With<OpenXRLeftController>,
         Without<OpenXRRightController>,
     )>,
     mut right_controller_query: Query<(
         &mut Transform,
+        Option<&mut AimPose>,
         With<OpenXRRightController>,
         Without<OpenXRLeftController>,
     )>,
@@ -61,8 +71,20 @@ pub fn update_open_xr_controllers(
     //get controller
     let controller = oculus_controller.get_ref(&instance, &session, &frame_state, &xr_input);
     //get left controller
-    let left = controller.grip_space(Hand::Left);
+    let left_grip_space = controller.grip_space(Hand::Left);
-    let left_postion = left.0.pose.position.to_vec3();
+    let left_aim_space = controller.aim_space(Hand::Left);
+    let left_postion = left_grip_space.0.pose.position.to_vec3();
+    let left_aim_pose = left_controller_query.get_single_mut().unwrap().1;
+    match left_aim_pose {
+        Some(mut pose) => {
+            *pose = AimPose(Transform {
+                translation: left_aim_space.0.pose.position.to_vec3(),
+                rotation: left_aim_space.0.pose.orientation.to_quat(),
+                scale: Vec3::splat(1.0),
+            });
+        }
+        None => (),
+    }
 
     left_controller_query
         .get_single_mut()
@@ -70,10 +92,24 @@ pub fn update_open_xr_controllers(
         .0
         .translation = left_postion;
 
-    left_controller_query.get_single_mut().unwrap().0.rotation = left.0.pose.orientation.to_quat();
+    left_controller_query.get_single_mut().unwrap().0.rotation =
+        left_grip_space.0.pose.orientation.to_quat();
     //get right controller
-    let right = controller.grip_space(Hand::Right);
+    let right_grip_space = controller.grip_space(Hand::Right);
-    let right_postion = right.0.pose.position.to_vec3();
+    let right_aim_space = controller.aim_space(Hand::Right);
+    let right_postion = right_grip_space.0.pose.position.to_vec3();
+
+    let right_aim_pose = right_controller_query.get_single_mut().unwrap().1;
+    match right_aim_pose {
+        Some(mut pose) => {
+            *pose = AimPose(Transform {
+                translation: right_aim_space.0.pose.position.to_vec3(),
+                rotation: right_aim_space.0.pose.orientation.to_quat(),
+                scale: Vec3::splat(1.0),
+            });
+        }
+        None => (),
+    }
 
     right_controller_query
         .get_single_mut()
@@ -82,6 +118,5 @@ pub fn update_open_xr_controllers(
         .translation = right_postion;
 
     right_controller_query.get_single_mut().unwrap().0.rotation =
-        right.0.pose.orientation.to_quat();
+        right_grip_space.0.pose.orientation.to_quat();
 }
-