embrace the egui

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "kneeboard2"
+version = "0.1.0"
+edition = "2024"
+description = "Bevy powered Kneeboard OpenXR Overlay"
+license = "MIT/Apache-2.0"
+
+[dependencies]
+bevy = { version = "0.18", features = ["debug"] }
+# bevy_cef = { path = "../bevy_cef" }
+# bevy_mod_xr = { git = "https://git.avii.nl/Avii/bevy_oxr.git", version = "0.5.0" }
+# bevy_mod_openxr = { git = "https://git.avii.nl/Avii/bevy_oxr.git", version = "0.5.0" }
+
+bevy_mod_xr = { path = "../bevy_oxr/crates/bevy_xr" }
+bevy_mod_openxr = { path = "../bevy_oxr/crates/bevy_openxr" }
+
+bevy_pkv = "0.15.0"
+openxr = "0.21.1"
+serde = { version = "1.0.228", features = ["derive"] }
+otd-ipc = { path = "../../otd-ipc-client-rs" }
+
+triple_buffer = "8.1.1"
+
+bevy_egui = "0.39.1"
+egui = "0.33.3"
+
+bms-briefing-parser = { git = "https://github.com/AviiNL/bms-kneeboard-server" }
+clap = { version = "4.5.60", features = ["derive"] }
+egui_extras = "0.33.3"
+encoding_rs_io = "0.1.7"
+encoding_rs = "0.8.35"

assets/shaders/shader.wgsl

@@ -0,0 +1,24 @@
+#import bevy_pbr::forward_io::{VertexOutput, FragmentOutput}
+
+@group(#{MATERIAL_BIND_GROUP}) @binding(101) var ui_texture: texture_2d<f32>;
+@group(#{MATERIAL_BIND_GROUP}) @binding(102) var ui_texture_sampler: sampler;
+@group(#{MATERIAL_BIND_GROUP}) @binding(103) var cursor_texture: texture_2d<f32>;
+@group(#{MATERIAL_BIND_GROUP}) @binding(104) var cursor_texture_sampler: sampler;
+
+@fragment
+fn fragment(
+    in: VertexOutput,
+) -> FragmentOutput {
+    var out: FragmentOutput;
+
+    // textureSample(material_color_texture, material_color_sampler, in.uv);
+
+    var surface = textureSample(ui_texture, ui_texture_sampler, in.uv);
+    var overlay = textureSample(cursor_texture, cursor_texture_sampler, in.uv);
+
+    var result_rgb = mix(surface.rgb, overlay.rgb, overlay.a);
+
+    out.color = vec4(result_rgb, 1.0);
+
+    return out;
+}

src/apad/custom_material.rs

@@ -0,0 +1,31 @@
+use bevy::{
+    asset::{Asset, Handle},
+    image::Image,
+    pbr::{ExtendedMaterial, MaterialExtension, StandardMaterial},
+    reflect::TypePath,
+    render::render_resource::AsBindGroup,
+    shader::ShaderRef,
+};
+
+const SHADER_ASSET_PATH: &str = "shaders/shader.wgsl";
+
+#[derive(Asset, TypePath, AsBindGroup, Debug, Clone)]
+pub struct CustomMaterial {
+    #[texture(101)]
+    #[sampler(102)]
+    pub ui_texture: Option<Handle<Image>>,
+
+    #[texture(103)]
+    #[sampler(104)]
+    pub cursor_texture: Option<Handle<Image>>,
+}
+
+/// The Material trait is very configurable, but comes with sensible defaults for all methods.
+/// You only need to implement functions for features that need non-default behavior. See the Material api docs for details!
+impl MaterialExtension for CustomMaterial {
+    fn fragment_shader() -> ShaderRef {
+        SHADER_ASSET_PATH.into()
+    }
+}
+
+pub type MyCustomMaterial = ExtendedMaterial<StandardMaterial, CustomMaterial>;

src/apad/mod.rs

@@ -0,0 +1,508 @@
+mod custom_material;
+mod keyboard;
+mod otdipcplugin;
+
+use std::f32::consts::{FRAC_PI_2, PI};
+
+use bevy::{
+    app::{App, Plugin, Startup, Update},
+    asset::{Assets, Handle},
+    camera::{Camera, Camera3d, RenderTarget, visibility::RenderLayers},
+    color::{Color, palettes::css},
+    ecs::{
+        component::Component,
+        entity::{ContainsEntity, Entity},
+        message::{MessageReader, MessageWriter},
+        query::{With, Without},
+        resource::Resource,
+        schedule::{IntoScheduleConfigs, ScheduleLabel},
+        system::{Commands, Query, Res, ResMut, Single},
+    },
+    image::Image,
+    math::{Dir3, Quat, Vec2, Vec3, primitives::Cuboid},
+    mesh::{Mesh, Mesh3d},
+    pbr::{MaterialPlugin, MeshMaterial3d, StandardMaterial},
+    prelude::{Deref, DerefMut},
+    render::render_resource::{Extent3d, TextureUsages},
+    transform::components::Transform,
+    utils::default,
+};
+use bevy_egui::{
+    EguiGlobalSettings, EguiInputSet, EguiMultipassSchedule, EguiPlugin, input::EguiInputEvent,
+};
+use bevy_mod_openxr::session::OxrSession;
+use bevy_pkv::{PersistentResourceAppExtensions, PkvStore};
+use egui::{Color32, Margin, PointerButton, Pos2};
+use openxr::Path;
+use serde::{Deserialize, Serialize};
+
+// PUB
+pub use crate::apad::otdipcplugin::TabletRotation;
+
+use crate::{
+    MainCamera,
+    apad::{
+        custom_material::{CustomMaterial, MyCustomMaterial},
+        keyboard::VirtualKeyboard,
+        otdipcplugin::{OtdIpcPlugin, PenButtons, PenDelta, PenPosition},
+    },
+    egui_pages::{Overview, overview},
+    vrcontrollerplugin::{
+        LeftController, LeftControllerActions, RightController, RightControllerActions,
+    },
+    vrplugin::Headset,
+};
+
+#[derive(ScheduleLabel, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct WorldspaceContextPass;
+
+#[derive(Resource, Deref)]
+pub struct TabletSize(Vec2);
+
+#[derive(Resource, Deref)]
+pub struct TabletResolutionScale(f32);
+
+#[derive(Resource, Deref, DerefMut)]
+struct PointerImage(Option<Handle<Image>>);
+
+#[derive(Component, Deref, DerefMut)]
+pub struct Keyboard(VirtualKeyboard);
+
+#[derive(Component)]
+pub struct Kneeboard;
+
+#[derive(Resource, Deref, DerefMut)]
+struct Name(String);
+
+#[derive(Resource, Deref, DerefMut)]
+struct KeyboardVisible(bool);
+
+#[derive(Resource, Deref, DerefMut)]
+struct WasPenButtonPressed(bool);
+
+#[derive(Resource, Deref, DerefMut)]
+struct PickedColor(Color32);
+
+#[derive(Component)]
+pub struct LookedAt;
+
+#[derive(Resource, Serialize, Deserialize)]
+pub struct KneeboardPosition {
+    position: Vec3,
+    rotation: Quat,
+}
+
+impl Default for KneeboardPosition {
+    fn default() -> Self {
+        Self {
+            position: Vec3::new(0.15, -0.25, -0.40),
+            rotation: Default::default(),
+        }
+    }
+}
+
+pub struct APadPlugin;
+impl Plugin for APadPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(OtdIpcPlugin)
+            .add_plugins(EguiPlugin::default())
+            .add_plugins(MaterialPlugin::<MyCustomMaterial>::default());
+
+        app.insert_resource(PkvStore::new("Avii", "Kneeboard"))
+            .init_persistent_resource::<KneeboardPosition>();
+
+        app.insert_resource(TabletSize(Vec2::new(210.0, 279.0)));
+        app.insert_resource(TabletResolutionScale(2.25));
+        app.insert_resource(PointerImage(None));
+        app.insert_resource(Name("".to_string()));
+        app.insert_resource(KeyboardVisible(false));
+        app.insert_resource(WasPenButtonPressed(false));
+        app.insert_resource(PickedColor(Color32::WHITE));
+
+        app.add_systems(Startup, (setup_pointer, setup_egui.after(setup_pointer)))
+            .add_systems(
+                Update,
+                (
+                    gaze,
+                    position_kneeboard,
+                    move_kneeboard,
+                    sync_camera_with_kneeboard,
+                ),
+            )
+            .add_systems(Update, pointer.in_set(EguiInputSet::InitReading))
+            .add_systems(WorldspaceContextPass, update);
+    }
+}
+
+fn setup_pointer(
+    mut images: ResMut<Assets<Image>>,
+    tablet_size: Res<TabletSize>,
+    tablet_res: Res<TabletResolutionScale>,
+    mut pointer: ResMut<PointerImage>,
+) {
+    let image = images.add({
+        let size = Extent3d {
+            width: (tablet_size.x * **tablet_res) as u32,
+            height: (tablet_size.y * **tablet_res) as u32,
+            depth_or_array_layers: 1,
+        };
+        let mut image = Image {
+            data: Some(vec![0; (size.width * size.height * 4) as usize]),
+            ..default()
+        };
+        image.texture_descriptor.usage |= TextureUsages::RENDER_ATTACHMENT;
+        image.texture_descriptor.size = size;
+        image
+    });
+
+    pointer.0 = Some(image);
+}
+
+fn pointer(
+    tablet_size: Res<TabletSize>,
+    tablet_res: Res<TabletResolutionScale>,
+    ent: Single<Entity, With<bevy_egui::EguiContext>>,
+    mesh_material: Single<&MeshMaterial3d<MyCustomMaterial>, With<Kneeboard>>,
+    mut pen_buttons: MessageReader<PenButtons>,
+    mut pen_delta: MessageReader<PenDelta>,
+    mut pen_position: MessageReader<PenPosition>,
+    mut input_writer: MessageWriter<EguiInputEvent>,
+    mut materials: ResMut<Assets<MyCustomMaterial>>,
+    mut images: ResMut<Assets<Image>>,
+    mut pointer: ResMut<PointerImage>,
+    mut pen_button_pressed: ResMut<WasPenButtonPressed>,
+) {
+    let Some(ref mut pointer) = pointer.0 else {
+        return;
+    };
+
+    let Some(image) = images.get_mut(&*pointer) else {
+        return;
+    };
+
+    image.clear(&[0u8; 4]);
+    for pos in pen_position.read() {
+        let x = pos.x * tablet_size.x * **tablet_res;
+        let y = pos.y * tablet_size.y * **tablet_res;
+
+        input_writer.write(EguiInputEvent {
+            context: *ent,
+            event: egui::Event::PointerMoved(Pos2::new(x, y)),
+        });
+
+        for button in pen_buttons.read() {
+            if button.a() && button.tip() {
+                for delta in pen_delta.read() {
+                    input_writer.write(EguiInputEvent {
+                        context: *ent,
+                        event: egui::Event::MouseWheel {
+                            unit: egui::MouseWheelUnit::Line,
+                            delta: egui::Vec2::new((-delta.x) * 20., (-delta.y) * 20.),
+                            modifiers: default(),
+                        },
+                    });
+                }
+
+                continue;
+            }
+
+            if **pen_button_pressed != button.tip() {
+                input_writer.write(EguiInputEvent {
+                    context: *ent,
+                    event: egui::Event::PointerButton {
+                        pos: Pos2::new(x, y),
+                        button: PointerButton::Primary,
+                        pressed: button.tip(),
+                        modifiers: Default::default(),
+                    },
+                });
+                **pen_button_pressed = button.tip();
+            }
+        }
+
+        draw_pointer(image, x, y);
+
+        if let Some(material) = materials.get_mut(mesh_material.0.id()) {
+            material.extension.cursor_texture = Some(pointer.clone());
+        }
+    }
+}
+
+fn draw_pointer(image: &mut Image, x: f32, y: f32) {
+    let x = x as i32;
+    let y = y as i32;
+
+    let distance = 3;
+    let length = 8;
+    // let thickness = 2;
+
+    for dx in (x - length - distance - 1)..=(x - distance - 1) {
+        let mut color = css::BLACK.into();
+        if dx == (x - length - distance - 1) || dx == (x - distance - 1) {
+            color = css::WHITE.into();
+        }
+
+        draw_pixel(image, dx, y - 1, css::WHITE.into());
+        draw_pixel(image, dx, y, color);
+        draw_pixel(image, dx, y + 1, css::WHITE.into());
+    }
+
+    for dx in (x + distance + 1)..=(x + length + distance + 1) {
+        let mut color = css::BLACK.into();
+        if dx == (x + distance + 1) || dx == (x + length + distance + 1) {
+            color = css::WHITE.into();
+        }
+
+        draw_pixel(image, dx, y - 1, css::WHITE.into());
+        draw_pixel(image, dx, y, color);
+        draw_pixel(image, dx, y + 1, css::WHITE.into());
+    }
+
+    for dy in (y - length - distance - 1)..=(y - distance - 1) {
+        let mut color = css::BLACK.into();
+        if dy == (y - length - distance - 1) || dy == (y - distance - 1) {
+            color = css::WHITE.into();
+        }
+
+        draw_pixel(image, x - 1, dy, css::WHITE.into());
+        draw_pixel(image, x, dy, color);
+        draw_pixel(image, x + 1, dy, css::WHITE.into());
+    }
+
+    for dy in (y + distance + 1)..=(y + length + distance + 1) {
+        let mut color = css::BLACK.into();
+        if dy == (y + distance + 1) || dy == (y + length + distance + 1) {
+            color = css::WHITE.into();
+        }
+
+        draw_pixel(image, x - 1, dy, css::WHITE.into());
+        draw_pixel(image, x, dy, color);
+        draw_pixel(image, x + 1, dy, css::WHITE.into());
+    }
+}
+
+fn draw_pixel(image: &mut Image, x: i32, y: i32, color: Color) {
+    if x < 0 || x > image.width() as i32 || y < 0 || y > image.height() as i32 {
+        return;
+    }
+
+    image.set_color_at(x as u32, y as u32, color).ok();
+}
+
+fn setup_egui(
+    mut commands: Commands,
+    mut images: ResMut<Assets<Image>>,
+    mut meshes: ResMut<Assets<Mesh>>,
+    mut materials: ResMut<Assets<MyCustomMaterial>>,
+    mut egui_global_settings: ResMut<EguiGlobalSettings>,
+    tablet_size: Res<TabletSize>,
+    tablet_res: Res<TabletResolutionScale>,
+    pointer: Res<PointerImage>,
+) {
+    egui_global_settings.auto_create_primary_context = false;
+    egui_global_settings.enable_cursor_icon_updates = false;
+
+    let image = images.add({
+        let size = Extent3d {
+            width: (tablet_size.x * **tablet_res) as u32,
+            height: (tablet_size.y * **tablet_res) as u32,
+            depth_or_array_layers: 1,
+        };
+        let mut image = Image {
+            data: Some(vec![0; (size.width * size.height * 4) as usize]),
+            ..default()
+        };
+        image.texture_descriptor.usage |= TextureUsages::RENDER_ATTACHMENT;
+        image.texture_descriptor.size = size;
+        image
+    });
+
+    commands.spawn((
+        Camera3d::default(),
+        RenderLayers::none(),
+        Camera::default(),
+        RenderTarget::Image(image.clone().into()),
+        EguiMultipassSchedule::new(WorldspaceContextPass),
+    ));
+
+    commands.spawn((
+        Mesh3d(meshes.add(Cuboid::new(
+            tablet_size.x / 1000.,
+            tablet_size.y / 1000.,
+            0.01,
+        ))),
+        MeshMaterial3d(materials.add(MyCustomMaterial {
+            base: StandardMaterial {
+                base_color: css::WHITE.into(),
+                unlit: true,
+                ..Default::default()
+            },
+            extension: CustomMaterial {
+                ui_texture: Some(image.clone()),
+                cursor_texture: pointer.clone(),
+            },
+        })),
+        Kneeboard,
+    ));
+
+    let keyboard = VirtualKeyboard::default();
+    commands.spawn(Keyboard(keyboard));
+}
+
+fn update(
+    mut ctx: Single<&mut bevy_egui::EguiContext>,
+    mut input: Single<&mut bevy_egui::EguiInput>,
+    mut keyboard: Single<&mut Keyboard>,
+
+    mo: Res<Overview>,
+) {
+    let bgcolor = egui::containers::Frame {
+        fill: egui::Color32::from_rgb(43, 44, 47),
+        inner_margin: Margin {
+            left: 15,
+            right: 15,
+            top: 15,
+            bottom: 15,
+        },
+        ..Default::default()
+    };
+    let focus = keyboard.is_active(ctx.get_mut());
+
+    let height = if focus { 200.0 } else { 0.0 };
+
+    egui::containers::CentralPanel::default()
+        .frame(bgcolor)
+        .show(ctx.get_mut(), |ui| {
+            egui::containers::TopBottomPanel::bottom("bottom_panel")
+                .resizable(false)
+                .height_range(egui::Rangef::new(0., height))
+                .show_inside(ui, |ui| {
+                    ui.vertical_centered(|ui| {
+                        keyboard.show(ui);
+                    });
+                });
+
+            egui::ScrollArea::vertical().show(ui, |ui| {
+                overview(ui, mo);
+
+                // ..
+            });
+        });
+
+    keyboard.bump_events(ctx.get_mut(), &mut input.0);
+}
+
+fn sync_camera_with_kneeboard(
+    kneeboard: Query<&Transform, With<Kneeboard>>,
+    mut cameras: Query<&mut Transform, (With<MainCamera>, Without<Kneeboard>)>,
+) {
+    let Ok(kneeboard) = kneeboard.single() else {
+        return;
+    };
+
+    let Ok(mut camera) = cameras.single_mut() else {
+        return;
+    };
+
+    let mut transform = *kneeboard;
+
+    transform.translation -= kneeboard.back().normalize() * 0.35;
+    transform.rotation *= Quat::from_axis_angle(Dir3::X.normalize(), PI)
+        * Quat::from_axis_angle(Dir3::Z.normalize(), PI);
+
+    *camera = transform;
+}
+
+fn gaze(
+    mut commands: Commands,
+    position: Res<KneeboardPosition>,
+    kneeboard: Single<Entity, With<Kneeboard>>,
+    head: Single<&Transform, With<Headset>>,
+) {
+    // let head = head;
+    // let kneeboard = kneeboard.single().expect("a kneeboard to exist");
+
+    let facing = head.forward().normalize();
+    let to_target = (position.position - head.translation).normalize();
+    let dot = facing.dot(to_target);
+    let cos_fov_angle: f32 = (35.0f32 / 2.0f32).to_radians().cos();
+
+    commands.entity(kneeboard.entity()).remove::<LookedAt>();
+    if dot < cos_fov_angle {
+        commands.entity(kneeboard.entity()).insert(LookedAt);
+    }
+}
+
+#[allow(clippy::type_complexity)]
+fn position_kneeboard(
+    mut transform: Query<(&mut Transform, Option<&LookedAt>), (With<Kneeboard>, Without<Headset>)>,
+    kneeboard: Res<KneeboardPosition>,
+    head: Query<&Transform, (With<Headset>, Without<Kneeboard>)>,
+) {
+    let Ok((mut transform, looked_at)) = transform.single_mut() else {
+        return;
+    };
+
+    let Ok(head) = head.single() else {
+        return;
+    };
+
+    transform.translation = kneeboard.position;
+    transform.rotation = kneeboard.rotation;
+
+    if looked_at.is_some() {
+        transform.translation = head.translation;
+        transform.rotation = head.rotation;
+    }
+}
+
+fn move_kneeboard(
+    session: Option<Res<OxrSession>>,
+    left: Option<Res<LeftControllerActions>>,
+    right: Option<Res<RightControllerActions>>,
+    left_transform: Query<&Transform, (With<LeftController>, Without<RightController>)>,
+    right_transform: Query<&Transform, (With<RightController>, Without<LeftController>)>,
+    mut kneeboard: ResMut<KneeboardPosition>,
+) {
+    let Some(session) = session else {
+        return;
+    };
+
+    let Some(left) = left else {
+        return;
+    };
+
+    let Some(right) = right else {
+        return;
+    };
+
+    let rot_offset: Quat = Quat::from_axis_angle(Vec3::new(1.0, 0.0, 0.0), FRAC_PI_2)
+        * Quat::from_axis_angle(Vec3::new(0.0, 0.0, 1.0), PI);
+
+    if let Ok(trigger_state) = left.squeeze_click.state(&session, Path::NULL)
+        && trigger_state.current_state
+    {
+        dbg!("squeeze triggered");
+
+        let Ok(transform) = left_transform.single() else {
+            return;
+        };
+
+        let pos_offset = transform.up().normalize() * -0.1;
+        kneeboard.position = transform.translation + pos_offset;
+        kneeboard.rotation = transform.rotation * rot_offset;
+    }
+
+    if let Ok(trigger_state) = right.squeeze_click.state(&session, Path::NULL)
+        && trigger_state.current_state
+    {
+        let Ok(transform) = right_transform.single() else {
+            return;
+        };
+
+        let pos_offset = transform.up().normalize() * -0.1;
+
+        kneeboard.position = transform.translation + pos_offset;
+        kneeboard.rotation = transform.rotation * rot_offset;
+    }
+}

src/apad/otdipcplugin.rs

@@ -0,0 +1,149 @@
+#![allow(unused)]
+
+use bevy::{
+    app::{App, Plugin, PreUpdate},
+    ecs::{
+        message::MessageWriter,
+        resource::Resource,
+        system::{Res, ResMut},
+    },
+    math::Vec2,
+    prelude::Deref,
+};
+use otd_ipc::{Message, OtdIpc};
+use triple_buffer::{Output, triple_buffer};
+
+#[derive(Resource)]
+struct OtdChannel(Output<Option<Message>>);
+
+#[derive(Resource)]
+struct LastPosition(Option<Vec2>);
+
+#[derive(Resource)]
+struct TabletSize(Option<Vec2>);
+
+#[derive(Resource)]
+struct MaxPenPressure(Option<u32>);
+
+#[derive(bevy::ecs::message::Message, Deref, Debug)]
+pub struct PenPosition(Vec2);
+
+#[derive(bevy::ecs::message::Message, Deref, Debug)]
+pub struct PenDelta(Vec2);
+
+#[derive(bevy::ecs::message::Message, Deref, Debug)]
+pub struct PenButtons(u32);
+
+impl PenButtons {
+    pub fn tip(&self) -> bool {
+        self.0 & 1 == 1
+    }
+
+    pub fn a(&self) -> bool {
+        self.0 & 2 == 2
+    }
+
+    pub fn b(&self) -> bool {
+        self.0 & 4 == 4
+    }
+}
+
+#[derive(bevy::ecs::message::Message)]
+pub struct PenPressure {
+    pub pressure: f32,
+}
+
+#[allow(clippy::upper_case_acronyms)]
+#[allow(unused)]
+#[derive(Resource)]
+pub enum TabletRotation {
+    Default,
+    CW,
+    UpsideDown,
+    CCW,
+}
+
+pub struct OtdIpcPlugin;
+
+impl Plugin for OtdIpcPlugin {
+    fn build(&self, app: &mut App) {
+        let (mut tx, rx) = triple_buffer(&None);
+        std::thread::spawn(move || {
+            let otd_ipc = OtdIpc::new("Kneeboard", "master").unwrap();
+
+            for msg in otd_ipc {
+                tx.write(Some(msg));
+            }
+        });
+        app.add_message::<PenDelta>();
+        app.add_message::<PenPosition>();
+        app.add_message::<PenPressure>();
+        app.add_message::<PenButtons>();
+        app.insert_resource(TabletRotation::Default);
+        app.insert_resource(OtdChannel(rx));
+        app.insert_resource(LastPosition(None));
+        app.insert_resource(MaxPenPressure(None));
+        app.insert_resource(TabletSize(None));
+        app.add_systems(PreUpdate, reader);
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn reader(
+    rotation: Res<TabletRotation>,
+    mut last_position: ResMut<LastPosition>,
+    mut channel: ResMut<OtdChannel>,
+    mut size: ResMut<TabletSize>,
+    mut pressure: ResMut<MaxPenPressure>,
+    mut pressure_writer: MessageWriter<PenPressure>,
+    mut position_writer: MessageWriter<PenPosition>,
+    mut delta_writer: MessageWriter<PenDelta>,
+    mut button_writer: MessageWriter<PenButtons>,
+) {
+    let Some(msg) = channel.0.read() else {
+        return;
+    };
+
+    match msg {
+        otd_ipc::Message::DeviceInfo(info) => {
+            size.0 = Some(Vec2::new(info.max_x, info.max_y));
+            pressure.0 = Some(info.max_pressure);
+        }
+        otd_ipc::Message::State(state) => {
+            let Some(size) = size.0 else {
+                return;
+            };
+
+            let Some(pressure) = pressure.0 else {
+                return;
+            };
+
+            button_writer.write(PenButtons(state.pen_buttons()));
+
+            let p = state.pressure() as f32 / pressure as f32;
+
+            pressure_writer.write(PenPressure { pressure: p });
+
+            let x = state.x();
+            let y = state.y();
+
+            let loc = match *rotation {
+                TabletRotation::Default => Vec2::new(x / size.x, y / size.y),
+                TabletRotation::CCW => Vec2::new(y / size.y, (size.x - x) / size.x),
+                TabletRotation::CW => Vec2::new((size.y - y) / size.y, x / size.x),
+                TabletRotation::UpsideDown => {
+                    Vec2::new((size.x - x) / size.x, (size.y - y) / size.y)
+                }
+            };
+
+            let lpos = last_position.0.unwrap_or(loc);
+
+            delta_writer.write(PenDelta(lpos - loc));
+
+            position_writer.write(PenPosition(loc));
+
+            last_position.0 = Some(loc);
+        }
+        _ => {}
+    }
+}

src/egui_pages/mod.rs

@@ -0,0 +1,80 @@
+mod overview;
+
+use std::{fs::File, io::Read, path::PathBuf};
+
+use bevy::{
+    app::{App, Plugin, Update},
+    ecs::{
+        resource::Resource,
+        system::{Res, ResMut},
+    },
+    time::{Time, Timer, TimerMode},
+};
+use encoding_rs::WINDOWS_1252;
+use encoding_rs_io::DecodeReaderBytesBuilder;
+pub use overview::*;
+
+use crate::BriefingPath;
+
+#[derive(Resource, Default)]
+struct RawBriefing(String);
+
+#[derive(Resource)]
+struct RefreshTimer(Timer);
+
+pub struct BmsPlugin {
+    briefing_path: PathBuf,
+}
+
+impl BmsPlugin {
+    pub fn new(briefing_path: PathBuf) -> Self {
+        Self { briefing_path }
+    }
+}
+
+impl Plugin for BmsPlugin {
+    fn build(&self, app: &mut App) {
+        app.init_resource::<Overview>();
+        app.init_resource::<RawBriefing>();
+
+        app.insert_resource(BriefingPath(self.briefing_path.clone()));
+
+        let mut timer = Timer::from_seconds(5.0, TimerMode::Repeating);
+        timer.almost_finish();
+        app.insert_resource(RefreshTimer(timer));
+
+        app.add_systems(Update, update_briefing);
+    }
+}
+
+fn update_briefing(
+    time: Res<Time>,
+    briefing_path: Res<BriefingPath>,
+    mut timer: ResMut<RefreshTimer>,
+    mut briefing: ResMut<RawBriefing>,
+    mut overview: ResMut<Overview>,
+) {
+    if !timer.0.tick(time.delta()).just_finished() {
+        return;
+    }
+
+    let mut path = briefing_path.clone();
+    path.push("briefing.txt");
+
+    let file = File::open(path).unwrap();
+
+    let mut buf = String::new();
+
+    DecodeReaderBytesBuilder::new()
+        .encoding(Some(WINDOWS_1252))
+        .build(file)
+        .read_to_string(&mut buf)
+        .unwrap();
+
+    if briefing.0 != buf {
+        *overview = bms_briefing_parser::Overview::from_briefing(&buf).into();
+
+        briefing.0 = buf;
+        println!("Updated...");
+    }
+}

src/egui_pages/overview.rs

@@ -0,0 +1,131 @@
+use bevy::ecs::{resource::Resource, system::Res};
+use egui::{Label, Ui};
+use egui_extras::*;
+
+#[derive(Resource, Default)]
+pub struct Overview {
+    pub callsign: String,
+    pub mission_type: String,
+    pub package_id: i32,
+    pub package_description: String,
+    pub package_mission: String,
+    pub target_area: String,
+    pub time_on_target: String,
+    pub sunrise: String,
+    pub sunset: String,
+}
+
+impl From<bms_briefing_parser::Overview<'_>> for Overview {
+    fn from(value: bms_briefing_parser::Overview) -> Self {
+        let sunrise = value
+            .sunrise
+            .replace('(', "\n")
+            .to_string()
+            .trim_end_matches(')')
+            .to_string();
+        let sunset = value
+            .sunset
+            .replace('(', "\n")
+            .trim_end_matches(')')
+            .to_string();
+
+        Self {
+            callsign: value.callsign.to_string(),
+            mission_type: value.mission_type.to_string(),
+            package_id: value.package_id,
+            package_description: value.package_description.to_string(),
+            package_mission: value.package_mission.to_string(),
+            target_area: value.target_area.to_string(),
+            time_on_target: value.time_on_target.to_string(),
+            sunrise,
+            sunset,
+        }
+    }
+}
+
+// use egui::Response;
+
+#[rustfmt::skip]
+pub fn overview(ui: &mut Ui, overview: Res<Overview>) {
+
+    let text_height = egui::TextStyle::Body
+            .resolve(ui.style())
+            .size
+            .max(ui.spacing().interact_size.y);
+
+
+    ui.vertical_centered(|ui| {
+        ui.heading(format!("Mission Overview - {}", overview.callsign));
+    });
+
+    let table = TableBuilder::new(ui)
+        .cell_layout(egui::Layout::left_to_right(egui::Align::Center))
+        .column(Column::remainder())
+        .column(Column::remainder())
+        .column(Column::remainder())
+        .column(Column::remainder());
+
+    table
+        .striped(true)
+        .cell_layout(egui::Layout::left_to_right(egui::Align::Center))
+        .header(20.0, |mut header| {
+            header.col(|ui| {
+                ui.strong("Mission Type");
+            });
+            header.col(|ui| {
+                ui.strong("Package #");
+            });
+            header.col(|ui| {
+                ui.strong("Description");
+            });
+            header.col(|ui| {
+                ui.strong("Mission");
+            });
+        })
+        .body(|mut body| {
+            body.row(text_height, |mut row| {
+                row.col(|ui| {
+                    ui.add(Label::new(&overview.mission_type).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+                row.col(|ui| {
+                    ui.add(Label::new(format!("{}", overview.package_id)).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+                row.col(|ui| {
+                    ui.add(Label::new(&overview.package_description).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+                row.col(|ui| {
+                    ui.add(Label::new(&overview.package_mission).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+            });
+
+            body.row(text_height, |mut row| {
+                row.col(|ui| {
+                    ui.strong("Target Area");
+                });
+                row.col(|ui| {
+                    ui.strong("Time on Target");
+                });
+                row.col(|ui| {
+                    ui.strong("Sunrise");
+                });
+                row.col(|ui| {
+                    ui.strong("Sunset");
+                });
+            });
+
+            body.row(text_height, |mut row| {
+                row.col(|ui| {
+                    ui.add(Label::new(&overview.target_area).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+                row.col(|ui| {
+                    ui.add(Label::new(&overview.time_on_target).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+                row.col(|ui| {
+                    ui.add(Label::new(&overview.sunrise).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+                row.col(|ui| {
+                    ui.add(Label::new(&overview.sunset).wrap_mode(egui::TextWrapMode::Wrap));
+                });
+            });
+        });
+}

src/main.rs

@@ -0,0 +1,77 @@
+#![allow(clippy::too_many_arguments)]
+
+mod apad;
+mod egui_pages;
+
+#[allow(unused)]
+mod vrcontrollerplugin;
+#[allow(unused)]
+mod vrplugin;
+
+use std::path::PathBuf;
+
+#[allow(unused)]
+use vrcontrollerplugin::VrControllersPlugin;
+#[allow(unused)]
+use vrplugin::VrPlugin;
+
+use bevy::{
+    prelude::*,
+    window::{PresentMode, WindowResolution},
+};
+use clap::Parser;
+
+use crate::{
+    apad::{APadPlugin, TabletRotation},
+    egui_pages::BmsPlugin,
+};
+
+#[derive(Component)]
+#[require(Camera3d)]
+pub struct MainCamera;
+
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+struct Args {
+    /// Directory containing briefing.txt
+    #[clap(trailing_var_arg = true)]
+    briefing_dir: PathBuf,
+}
+
+#[derive(Resource, Deref)]
+pub struct BriefingPath(PathBuf);
+
+fn main() {
+    let args = Args::parse();
+
+    App::new()
+        // .add_plugins(DefaultPlugins)
+        .add_plugins(VrPlugin)
+        .add_plugins(VrControllersPlugin)
+        .add_plugins(BmsPlugin::new(args.briefing_dir))
+        .add_plugins(APadPlugin)
+        .add_systems(Startup, setup)
+        .insert_resource(TabletRotation::CW)
+        .run();
+}
+
+fn setup(mut commands: Commands, mut window: Query<&mut Window>) {
+    let Ok(mut window) = window.single_mut() else {
+        return;
+    };
+
+    window.title = "Kneeboard".to_string();
+    window.resolution = WindowResolution::new(550, 720);
+    window.position = WindowPosition::new(IVec2::new(2560 + 1280, 600));
+    window.present_mode = PresentMode::AutoNoVsync;
+    window.fit_canvas_to_parent = false;
+    window.prevent_default_event_handling = false;
+
+    commands.spawn((
+        Camera {
+            clear_color: ClearColorConfig::Custom(Color::NONE),
+            ..default()
+        },
+        MainCamera,
+    ));
+}

src/vrcontrollerplugin.rs

@@ -0,0 +1,237 @@
+use bevy::prelude::*;
+use bevy_mod_openxr::prelude::*;
+use bevy_mod_xr::session::{XrSessionCreated, session_available};
+use openxr::{Action, Posef, Vector2f};
+
+#[derive(Resource)]
+pub struct LeftControllerActions {
+    set: openxr::ActionSet,
+    pose: Action<Posef>,
+    pub thumbstick: Action<Vector2f>,
+    pub trigger: Action<f32>,
+    pub trigger_click: Action<bool>,
+    pub squeeze: Action<f32>,
+    pub squeeze_click: Action<bool>,
+}
+
+#[derive(Resource)]
+pub struct RightControllerActions {
+    set: openxr::ActionSet,
+    pose: Action<Posef>,
+    pub thumbstick: Action<Vector2f>,
+    pub trigger: Action<f32>,
+    pub trigger_click: Action<bool>,
+    pub squeeze: Action<f32>,
+    pub squeeze_click: Action<bool>,
+}
+
+#[derive(Component)]
+pub struct LeftController;
+
+#[derive(Component)]
+pub struct RightController;
+
+pub struct VrControllersPlugin;
+
+impl Plugin for VrControllersPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_systems(XrSessionCreated, spawn_left_controller);
+        app.add_systems(XrSessionCreated, spawn_right_controller);
+
+        app.add_systems(XrSessionCreated, attach_set);
+        app.add_systems(
+            PreUpdate,
+            sync_actions
+                .before(OxrActionSetSyncSet)
+                .run_if(openxr_session_running),
+        );
+
+        app.add_systems(Startup, create_actions_left.run_if(session_available));
+        app.add_systems(Startup, create_actions_right.run_if(session_available));
+
+        app.add_systems(
+            OxrSendActionBindings,
+            suggest_action_bindings_left.after(create_actions_left),
+        );
+        app.add_systems(
+            OxrSendActionBindings,
+            suggest_action_bindings_right.after(create_actions_right),
+        );
+    }
+}
+
+fn attach_set(
+    left: Res<LeftControllerActions>,
+    right: Res<RightControllerActions>,
+    mut attach: MessageWriter<OxrAttachActionSet>,
+) {
+    attach.write(OxrAttachActionSet(left.set.clone()));
+    attach.write(OxrAttachActionSet(right.set.clone()));
+}
+
+fn sync_actions(
+    left: Res<LeftControllerActions>,
+    right: Res<RightControllerActions>,
+    mut sync: MessageWriter<OxrSyncActionSet>,
+) {
+    sync.write(OxrSyncActionSet(left.set.clone()));
+    sync.write(OxrSyncActionSet(right.set.clone()));
+}
+
+fn suggest_action_bindings_left(
+    actions: Res<LeftControllerActions>,
+    mut bindings: MessageWriter<OxrSuggestActionBinding>,
+) {
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.pose.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/grip/pose".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/trigger/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/trigger/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/squeeze/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/squeeze/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.thumbstick.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/left/input/thumbstick".into()],
+    });
+}
+
+fn suggest_action_bindings_right(
+    actions: Res<RightControllerActions>,
+    mut bindings: MessageWriter<OxrSuggestActionBinding>,
+) {
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.pose.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/grip/pose".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/trigger/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.trigger_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/trigger/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/squeeze/value".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.squeeze_click.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/squeeze/click".into()],
+    });
+
+    bindings.write(OxrSuggestActionBinding {
+        action: actions.thumbstick.as_raw(),
+        interaction_profile: "/interaction_profiles/bytedance/pico4_controller".into(),
+        bindings: vec!["/user/hand/right/input/thumbstick".into()],
+    });
+}
+
+fn create_actions_left(instance: Res<OxrInstance>, mut cmds: Commands) {
+    let set = instance
+        .create_action_set("left-controller", "Left Controller", 0)
+        .unwrap();
+    let pose = set.create_action("pose", "Grip Pose", &[]).unwrap();
+    let trigger = set.create_action("trigger", "Trigger", &[]).unwrap();
+    let trigger_click = set
+        .create_action("trigger_click", "Trigger Click", &[])
+        .unwrap();
+    let squeeze = set.create_action("squeeze", "Squeeze", &[]).unwrap();
+    let squeeze_click = set
+        .create_action("squeeze_click", "Squeeze Click", &[])
+        .unwrap();
+    let thumbstick = set.create_action("thumbstick", "Thumbstick", &[]).unwrap();
+
+    cmds.insert_resource(LeftControllerActions {
+        set,
+        pose,
+        thumbstick,
+        trigger,
+        trigger_click,
+        squeeze,
+        squeeze_click,
+    })
+}
+
+fn create_actions_right(instance: Res<OxrInstance>, mut cmds: Commands) {
+    let set = instance
+        .create_action_set("right-controller", "Right Controller", 0)
+        .unwrap();
+    let pose = set.create_action("pose", "Grip Pose", &[]).unwrap();
+    let trigger = set.create_action("trigger", "Trigger", &[]).unwrap();
+    let trigger_click = set
+        .create_action("trigger_click", "Trigger Click", &[])
+        .unwrap();
+    let squeeze = set.create_action("squeeze", "Squeeze", &[]).unwrap();
+    let squeeze_click = set
+        .create_action("squeeze_click", "Squeeze Click", &[])
+        .unwrap();
+    let thumbstick = set.create_action("thumbstick", "Thumbstick", &[]).unwrap();
+
+    cmds.insert_resource(RightControllerActions {
+        set,
+        pose,
+        thumbstick,
+        trigger,
+        trigger_click,
+        squeeze,
+        squeeze_click,
+    })
+}
+
+fn spawn_left_controller(
+    left: Res<LeftControllerActions>,
+    mut commands: Commands,
+    session: Res<OxrSession>,
+) {
+    let space = session
+        .create_action_space(&left.pose, openxr::Path::NULL, Isometry3d::IDENTITY)
+        .unwrap();
+
+    commands.spawn((space, LeftController));
+}
+
+fn spawn_right_controller(
+    right: Res<RightControllerActions>,
+    mut commands: Commands,
+    session: Res<OxrSession>,
+) {
+    let space = session
+        .create_action_space(&right.pose, openxr::Path::NULL, Isometry3d::IDENTITY)
+        .unwrap();
+
+    commands.spawn((space, RightController));
+}

src/vrplugin.rs

@@ -0,0 +1,49 @@
+use bevy::{
+    prelude::*,
+    render::render_resource::TextureFormat,
+    window::{PresentMode, WindowResolution},
+};
+use bevy_mod_openxr::prelude::*;
+use bevy_mod_xr::session::XrSessionCreated;
+
+use crate::vrcontrollerplugin::VrControllersPlugin;
+
+#[derive(Component)]
+pub struct Headset;
+
+pub struct VrPlugin;
+
+impl Plugin for VrPlugin {
+    fn build(&self, app: &mut bevy::app::App) {
+        app.add_plugins(
+            add_xr_plugins(DefaultPlugins)
+                .disable::<HandTrackingPlugin>()
+                .build()
+                .set(OxrInitPlugin {
+                    exts: {
+                        let mut exts = OxrExtensions::default();
+                        exts.extx_overlay = true;
+                        exts
+                    },
+                    ..Default::default()
+                }),
+        );
+
+        app.insert_resource(ClearColor(Color::NONE))
+            .insert_resource(OxrSessionConfig {
+                blend_mode_preference: { vec![EnvironmentBlendMode::ALPHA_BLEND] },
+                formats: Some(vec![TextureFormat::Rgba8UnormSrgb]),
+                ..Default::default()
+            });
+
+        app.add_systems(XrSessionCreated, create_view_space);
+    }
+}
+
+pub fn create_view_space(session: Res<OxrSession>, mut commands: Commands) {
+    let space = session
+        .create_reference_space(openxr::ReferenceSpaceType::VIEW, Isometry3d::IDENTITY)
+        .unwrap();
+
+    commands.spawn((Headset, space.0));
+}