Initial Commit

.gitignore

@@ -0,0 +1,7 @@
+target/
+.idea/
+book/
+.DS_Store**/.DS_Store
+.claude/memo.md
+docs/plans/
+.worktrees/

Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "kneeboard"
+version = "0.1.0"
+edition = "2024"
+description = "Bevy powered Kneeboard OpenXR Overlay"
+license = "MIT/Apache-2.0"
+
+[dependencies]
+avian3d = { version = "0.5.0", features = ["simd"] }
+bevy = { version = "0.18", features = ["debug"] }
+bevy_cef = { path = "../bevy_cef" }
+bevy_mod_xr = { git = "https://git.avii.nl/Avii/bevy_oxr.git", version = "0.5.0" }
+bevy_mod_openxr = { git = "https://git.avii.nl/Avii/bevy_oxr.git", version = "0.5.0" }
+bevy_pkv = "0.15.0"
+# bevy_xr_utils.workspace = true
+openxr = "0.21.1"
+rdev = { version = "0.5.3", features = ["unstable_grab"] }
+serde = { version = "1.0.228", features = ["derive"] }

src/kneeboardplugin.rs

@@ -0,0 +1,197 @@
+use std::f32::consts::PI;
+
+use avian3d::{math::FRAC_PI_2, prelude::*};
+use bevy::prelude::*;
+use bevy_cef::prelude::*;
+use bevy_mod_openxr::prelude::*;
+use bevy_mod_xr::session::XrSessionCreated;
+use bevy_pkv::{PersistentResourceAppExtensions, PkvStore};
+use openxr::Path;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    vrcontrollerplugin::{
+        LeftController, LeftControllerActions, RightController, RightControllerActions,
+    },
+    vrplugin::{Headset, MainCamera, create_view_space},
+};
+
+#[derive(Component)]
+pub struct Kneeboard;
+
+#[derive(Resource, Serialize, Deserialize)]
+pub struct KneeboardPosition {
+    position: Vec3,
+    rotation: Quat,
+}
+
+impl Default for KneeboardPosition {
+    fn default() -> Self {
+        Self {
+            position: Vec3::new(0.15, -0.25, -0.40),
+            rotation: Default::default(),
+        }
+    }
+}
+
+pub struct KneeboardPlugin;
+
+impl Plugin for KneeboardPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(PhysicsPlugins::default());
+        app.add_plugins(CefPlugin {
+            command_line_config: CommandLineConfig {
+                switches: ["--no-zygote", "--no-sandbox"].to_vec(),
+                ..Default::default()
+            },
+            ..Default::default()
+        });
+        app.insert_resource(PkvStore::new("Avii", "Kneeboard"))
+            .init_persistent_resource::<KneeboardPosition>();
+        app.add_systems(XrSessionCreated, spawn_kneeboard.after(create_view_space));
+        app.add_systems(Update, gaze.run_if(openxr_session_running));
+        app.add_systems(Update, move_keyboard.run_if(openxr_session_running));
+        app.add_systems(Update, position_kneeboard.after(move_keyboard));
+        app.add_systems(Update, sync_camera_with_kneeboard.after(position_kneeboard));
+        app.add_systems(Update, scroll_handler.run_if(openxr_session_running));
+    }
+}
+
+fn sync_camera_with_kneeboard(
+    kneeboard: Query<&Transform, With<Kneeboard>>,
+    mut cameras: Query<&mut Transform, (With<MainCamera>, Without<Kneeboard>)>,
+) {
+    let Ok(kneeboard) = kneeboard.single() else {
+        return;
+    };
+
+    let Ok(mut camera) = cameras.single_mut() else {
+        return;
+    };
+
+    let mut transform = *kneeboard;
+
+    transform.translation -= kneeboard.back().normalize() * 0.35;
+    transform.rotation *= Quat::from_axis_angle(Dir3::X.normalize(), PI)
+        * Quat::from_axis_angle(Dir3::Z.normalize(), PI);
+
+    *camera = transform;
+}
+
+fn scroll_handler(
+    session: Res<OxrSession>,
+    right: Res<RightControllerActions>,
+    browsers: NonSend<Browsers>,
+    webviews: Query<Entity, With<WebviewSource>>,
+) {
+    let Ok(thumbstick) = right.thumbstick.state(&session, Path::NULL) else {
+        return;
+    };
+
+    let delta = Vec2::new(thumbstick.current_state.x, thumbstick.current_state.y) * 8.0;
+
+    for webview in webviews.iter() {
+        browsers.send_mouse_wheel(&webview, Vec2::ZERO, delta);
+    }
+}
+
+fn position_kneeboard(
+    mut transform: Query<&mut Transform, With<Kneeboard>>,
+    kneeboard: Res<KneeboardPosition>,
+) {
+    let Ok(mut transform) = transform.single_mut() else {
+        return;
+    };
+
+    transform.translation = kneeboard.position;
+    transform.rotation = kneeboard.rotation;
+}
+
+fn move_keyboard(
+    session: Res<OxrSession>,
+    left: Res<LeftControllerActions>,
+    right: Res<RightControllerActions>,
+    left_transform: Query<&Transform, (With<LeftController>, Without<RightController>)>,
+    right_transform: Query<&Transform, (With<RightController>, Without<LeftController>)>,
+    mut kneeboard: ResMut<KneeboardPosition>,
+) {
+    let rot_offset: Quat = Quat::from_axis_angle(Vec3::new(1.0, 0.0, 0.0), FRAC_PI_2)
+        * Quat::from_axis_angle(Vec3::new(0.0, 0.0, 1.0), PI);
+
+    if let Ok(trigger_state) = left.squeeze_click.state(&session, Path::NULL)
+        && trigger_state.current_state
+    {
+        let Ok(transform) = left_transform.single() else {
+            return;
+        };
+
+        let pos_offset = transform.up().normalize() * -0.1;
+        kneeboard.position = transform.translation + pos_offset;
+        kneeboard.rotation = transform.rotation * rot_offset;
+    }
+
+    if let Ok(trigger_state) = right.squeeze_click.state(&session, Path::NULL)
+        && trigger_state.current_state
+    {
+        let Ok(transform) = right_transform.single() else {
+            return;
+        };
+
+        let pos_offset = transform.up().normalize() * -0.1;
+
+        kneeboard.position = transform.translation + pos_offset;
+        kneeboard.rotation = transform.rotation * rot_offset;
+    }
+}
+
+fn spawn_kneeboard(
+    mut commands: Commands,
+    mut meshes: ResMut<Assets<Mesh>>,
+    mut materials: ResMut<Assets<WebviewExtendStandardMaterial>>,
+    position: Res<KneeboardPosition>,
+) {
+    commands.spawn((
+        RigidBody::Static,
+        WebviewSource::new("http://localhost:7878/"),
+        WebviewSize(Vec2::new(210.0 * 3.5, 279.0 * 3.5)),
+        Collider::cuboid(0.210, 0.279, 0.01),
+        CollidingEntities::default(),
+        Mesh3d(meshes.add(Cuboid::new(0.210, 0.279, 0.01))),
+        MeshMaterial3d(materials.add(WebviewExtendStandardMaterial {
+            base: StandardMaterial {
+                unlit: true,
+                ..default()
+            },
+            ..Default::default()
+        })),
+        Transform::from_translation(position.position).rotate(position.rotation),
+        Kneeboard,
+    ));
+}
+
+fn gaze(
+    material_handles: Query<&MeshMaterial3d<WebviewExtendStandardMaterial>, With<Kneeboard>>,
+    mut materials: ResMut<Assets<WebviewExtendStandardMaterial>>,
+    position: Res<KneeboardPosition>,
+    head_query: Query<&Transform, With<Headset>>,
+) {
+    let head = head_query.single().expect("a head to exist");
+
+    let facing = head.forward().normalize();
+    let to_target = (position.position - head.translation).normalize();
+    let dot = facing.dot(to_target);
+    let cos_fov_angle: f32 = (35.0f32 / 2.0f32).to_radians().cos();
+
+    let mut alpha = 1.0;
+    if dot < cos_fov_angle {
+        alpha = 0.0;
+    }
+
+    for material_handle in material_handles.iter() {
+        if let Some(material) = materials.get_mut(material_handle)
+            && let Color::LinearRgba(ref mut rgba) = material.base.base_color
+        {
+            rgba.alpha = alpha;
+        }
+    }
+}

src/main.rs

@@ -0,0 +1,25 @@
+//! A simple 3D scene with light shining over a cube sitting on a plane.
+
+mod kneeboardplugin;
+mod vrcontrollerplugin;
+mod vrplugin;
+
+use vrplugin::VrPlugin;
+
+use bevy::prelude::*;
+
+use crate::{kneeboardplugin::KneeboardPlugin, vrcontrollerplugin::VrControllersPlugin};
+
+fn main() {
+    App::new()
+        .insert_resource(ClearColor(Color::linear_rgb(1.0, 0.0, 1.0)))
+        .add_plugins(VrPlugin)
+        .add_plugins(VrControllersPlugin)
+        .add_plugins(KneeboardPlugin)
+        .insert_resource(ClearColor(Color::NONE))
+        // .insert_resource(GlobalAmbientLight {
+        //     brightness: 1000.0,
+        //     ..GlobalAmbientLight::default()
+        // })
+        .run();
+}

src/vrcontrollerplugin.rs

@@ -0,0 +1,231 @@
+use bevy::prelude::*;
+use bevy_mod_openxr::prelude::*;
+use bevy_mod_xr::session::{XrSessionCreated, session_available};
+use openxr::{Action, Posef, Vector2f};
+
+#[derive(Resource)]
+pub struct LeftControllerActions {
+    set: openxr::ActionSet,
+    pose: Action<Posef>,
+    pub thumbstick: Action<Vector2f>,
+    pub trigger: Action<f32>,
+    pub trigger_click: Action<bool>,
+    pub squeeze: Action<f32>,
+    pub squeeze_click: Action<bool>,
+}
+
+#[derive(Resource)]
+pub struct RightControllerActions {
+    set: openxr::ActionSet,
+    pose: Action<Posef>,
+    pub thumbstick: Action<Vector2f>,
+    pub trigger: Action<f32>,
+    pub trigger_click: Action<bool>,
+    pub squeeze: Action<f32>,
+    pub squeeze_click: Action<bool>,
+}
+
+#[derive(Component)]
+pub struct LeftController;
+
+#[derive(Component)]
+pub struct RightController;
+
+pub struct VrControllersPlugin;
+
+impl Plugin for VrControllersPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(XrSessionCreated, spawn_left_controller);
+        app.add_systems(XrSessionCreated, spawn_right_controller);
+
+        app.add_systems(XrSessionCreated, attach_set);
+        app.add_systems(
+            PreUpdate,
+            sync_actions
+                .before(OxrActionSetSyncSet)
+                .run_if(openxr_session_running),
+        );
+
+        app.add_systems(OxrSendActionBindings, suggest_action_bindings_left);
+        app.add_systems(OxrSendActionBindings, suggest_action_bindings_right);
+
+        app.add_systems(Startup, create_actions_left.run_if(session_available));
+        app.add_systems(Startup, create_actions_right.run_if(session_available));
+    }
+}
+
+fn attach_set(
+    left: Res<LeftControllerActions>,
+    right: Res<RightControllerActions>,
+    mut attach: MessageWriter<OxrAttachActionSet>,
+) {
+    attach.write(OxrAttachActionSet(left.set.clone()));
+    attach.write(OxrAttachActionSet(right.set.clone()));
+}
+
+fn sync_actions(
+    left: Res<LeftControllerActions>,
+    right: Res<RightControllerActions>,
+    mut sync: MessageWriter<OxrSyncActionSet>,
+) {
+    sync.write(OxrSyncActionSet(left.set.clone()));
+    sync.write(OxrSyncActionSet(right.set.clone()));
+}
+
+fn suggest_action_bindings_left(
+    actions: Res<LeftControllerActions>,
+    mut bindings: MessageWriter<OxrSuggestActionBinding>,
+) {
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.pose.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/grip/pose".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/trigger/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/trigger/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/squeeze/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/squeeze/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.thumbstick.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/thumbstick".into()],
+    });
+}
+
+fn suggest_action_bindings_right(
+    actions: Res<RightControllerActions>,
+    mut bindings: MessageWriter<OxrSuggestActionBinding>,
+) {
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.pose.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/grip/pose".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/trigger/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/trigger/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/squeeze/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/squeeze/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.thumbstick.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/thumbstick".into()],
+    });
+}
+
+fn create_actions_left(instance: Res<OxrInstance>, mut cmds: Commands) {
+    let set = instance
+        .create_action_set("left-controller", "Left Controller", 0)
+        .unwrap();
+    let pose = set.create_action("pose", "Grip Pose", &[]).unwrap();
+    let trigger = set.create_action("trigger", "Trigger", &[]).unwrap();
+    let trigger_click = set
+        .create_action("trigger_click", "Trigger Click", &[])
+        .unwrap();
+    let squeeze = set.create_action("squeeze", "Squeeze", &[]).unwrap();
+    let squeeze_click = set
+        .create_action("squeeze_click", "Squeeze Click", &[])
+        .unwrap();
+    let thumbstick = set.create_action("thumbstick", "Thumbstick", &[]).unwrap();
+
+    cmds.insert_resource(LeftControllerActions {
+        set,
+        pose,
+        thumbstick,
+        trigger,
+        trigger_click,
+        squeeze,
+        squeeze_click,
+    })
+}
+
+fn create_actions_right(instance: Res<OxrInstance>, mut cmds: Commands) {
+    let set = instance
+        .create_action_set("right-controller", "Right Controller", 0)
+        .unwrap();
+    let pose = set.create_action("pose", "Grip Pose", &[]).unwrap();
+    let trigger = set.create_action("trigger", "Trigger", &[]).unwrap();
+    let trigger_click = set
+        .create_action("trigger_click", "Trigger Click", &[])
+        .unwrap();
+    let squeeze = set.create_action("squeeze", "Squeeze", &[]).unwrap();
+    let squeeze_click = set
+        .create_action("squeeze_click", "Squeeze Click", &[])
+        .unwrap();
+    let thumbstick = set.create_action("thumbstick", "Thumbstick", &[]).unwrap();
+
+    cmds.insert_resource(RightControllerActions {
+        set,
+        pose,
+        thumbstick,
+        trigger,
+        trigger_click,
+        squeeze,
+        squeeze_click,
+    })
+}
+
+fn spawn_left_controller(
+    left: Res<LeftControllerActions>,
+    mut commands: Commands,
+    session: Res<OxrSession>,
+) {
+    let space = session
+        .create_action_space(&left.pose, openxr::Path::NULL, Isometry3d::IDENTITY)
+        .unwrap();
+
+    commands.spawn((space, LeftController));
+}
+
+fn spawn_right_controller(
+    right: Res<RightControllerActions>,
+    mut commands: Commands,
+    session: Res<OxrSession>,
+) {
+    let space = session
+        .create_action_space(&right.pose, openxr::Path::NULL, Isometry3d::IDENTITY)
+        .unwrap();
+
+    commands.spawn((space, RightController));
+}

src/vrplugin.rs

@@ -0,0 +1,69 @@
+use bevy::prelude::*;
+use bevy_mod_openxr::prelude::*;
+use bevy_mod_xr::session::XrSessionCreated;
+
+#[derive(Component)]
+#[require(Camera3d)]
+pub struct MainCamera;
+
+#[derive(Component)]
+pub struct Headset;
+
+pub struct VrPlugin;
+
+impl Plugin for VrPlugin {
+    fn build(&self, app: &mut bevy::app::App) {
+        app.add_plugins(
+            add_xr_plugins(DefaultPlugins)
+                .disable::<HandTrackingPlugin>()
+                .build()
+                .set(OxrInitPlugin {
+                    exts: {
+                        let mut exts = OxrExtensions::default();
+                        exts.extx_overlay = true;
+                        exts
+                    },
+                    ..Default::default()
+                }),
+        );
+
+        app.insert_resource(OxrSessionConfig {
+            blend_mode_preference: {
+                vec![
+                    EnvironmentBlendMode::ALPHA_BLEND,
+                    EnvironmentBlendMode::ADDITIVE,
+                    EnvironmentBlendMode::OPAQUE,
+                ]
+            },
+            ..OxrSessionConfig::default()
+        });
+
+        app.add_systems(XrSessionCreated, create_view_space);
+        // .add_systems(Update, sync_head_with_camera.run_if(openxr_session_running));
+    }
+}
+
+pub fn create_view_space(session: Res<OxrSession>, mut commands: Commands) {
+    let space = session
+        .create_reference_space(openxr::ReferenceSpaceType::VIEW, Isometry3d::IDENTITY)
+        .unwrap();
+
+    commands.spawn((Headset, space.0));
+    commands.spawn((
+        Camera {
+            clear_color: ClearColorConfig::Custom(Color::linear_rgb(1.0, 0.0, 1.0)),
+            ..default()
+        },
+        MainCamera,
+        Transform::from_xyz(-2.5, 4.5, 9.0).looking_at(Vec3::ZERO, Vec3::Y),
+    ));
+}
+
+// fn sync_head_with_camera(
+//     heads: Query<&Transform, With<Headset>>,
+//     cameras: Query<&mut Transform, (With<MainCamera>, Without<Headset>)>,
+// ) {
+//     for mut camera in cameras {
+//         *camera = *heads.single().unwrap();
+//     }
+// }