From bb05dc51610470efac867bf37e610745c9612a63 Mon Sep 17 00:00:00 2001
From: Avii <git@avii.nl>
Date: Sat, 21 Feb 2026 19:08:35 +0100
Subject: [PATCH] head lookat move mouse test

---
 src/main.rs | 80 +++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 74 insertions(+), 6 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index d5d490b..947de3d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -4,21 +4,89 @@ mod kneeboardplugin;
 mod vrcontrollerplugin;
 mod vrplugin;
 
+use bevy_cef::prelude::{Browsers, MeshAabb, WebviewSize, WebviewSource};
+use bevy_mod_openxr::openxr_session_running;
+use bevy_mod_xr::camera::XrCamera;
 use vrplugin::VrPlugin;
 
 use bevy::prelude::*;
 
-use crate::{kneeboardplugin::KneeboardPlugin, vrcontrollerplugin::VrControllersPlugin};
+use crate::{
+    kneeboardplugin::{Kneeboard, KneeboardPlugin, LookedAt},
+    vrcontrollerplugin::VrControllersPlugin,
+};
 
 fn main() {
     App::new()
-        .add_plugins(VrPlugin)
+        .add_plugins((VrPlugin, MeshPickingPlugin))
         .add_plugins(VrControllersPlugin)
         .add_plugins(KneeboardPlugin)
         .insert_resource(ClearColor(Color::NONE))
-        // .insert_resource(GlobalAmbientLight {
-        //     brightness: 1000.0,
-        //     ..GlobalAmbientLight::default()
-        // })
+        .add_systems(Update, head_pointer.run_if(openxr_session_running))
         .run();
 }
+
+#[allow(clippy::type_complexity)]
+fn head_pointer(
+    browsers: NonSend<Browsers>,
+    webviews: Query<(Entity, &WebviewSize), With<WebviewSource>>,
+    headset: Query<&GlobalTransform, With<XrCamera>>,
+    kneeboard: Query<(&GlobalTransform, Option<&LookedAt>), With<Kneeboard>>,
+    aabb: MeshAabb,
+) {
+    let Ok((webview, size)) = webviews.single() else {
+        return;
+    };
+
+    let tex_size = size.0;
+
+    if let Some(gt) = headset.into_iter().next() {
+        let Ok((plane_tf, looked_at)) = kneeboard.single() else {
+            return;
+        };
+
+        if looked_at.is_some() {
+            // this is inverted for some reason wtf
+            return;
+        }
+
+        let (min, max) = aabb.calculate_local(webview);
+        let plane_size = Vec2::new(max.x - min.x, max.y - min.y);
+
+        let ray = Ray3d::new(gt.translation(), gt.forward());
+
+        let n = plane_tf.forward().as_vec3();
+        let Some(t) = ray.intersect_plane(
+            plane_tf.translation(),
+            InfinitePlane3d::new(plane_tf.forward()),
+        ) else {
+            return;
+        };
+        let hit_world = ray.origin + ray.direction * t;
+        let local_hit = plane_tf.affine().inverse().transform_point(hit_world);
+        let local_normal = plane_tf.affine().inverse().transform_vector3(n).normalize();
+        let abs_normal = local_normal.abs();
+        let (u_coord, v_coord) = if abs_normal.z > abs_normal.x && abs_normal.z > abs_normal.y {
+            (local_hit.x, local_hit.y)
+        } else if abs_normal.y > abs_normal.x {
+            (local_hit.x, local_hit.z)
+        } else {
+            (local_hit.y, local_hit.z)
+        };
+
+        let w = plane_size.x;
+        let h = plane_size.y;
+        let u = (u_coord + w * 0.5) / w;
+        let v = (v_coord + h * 0.5) / h;
+        if !(0.0..=1.0).contains(&u) || !(0.0..=1.0).contains(&v) {
+            // outside plane bounds
+            return;
+        }
+        let px = (1.0 - u) * tex_size.x;
+        let py = (1.0 - v) * tex_size.y;
+
+        let pos = Vec2::new(px, py);
+
+        browsers.send_mouse_move(&webview, &[], pos, false);
+    }
+}