mod custom_material; mod keyboard; mod otdipcplugin; use std::f32::consts::{FRAC_PI_2, PI}; use bevy::{ app::{App, Plugin, Startup, Update}, asset::{Assets, Handle}, camera::{Camera, Camera3d, RenderTarget, visibility::RenderLayers}, color::{Color, palettes::css}, ecs::{ component::Component, entity::{ContainsEntity, Entity}, message::{MessageReader, MessageWriter}, query::{With, Without}, resource::Resource, schedule::{IntoScheduleConfigs, ScheduleLabel}, system::{Commands, Query, Res, ResMut, Single}, }, image::Image, math::{Dir3, Quat, Vec2, Vec3, primitives::Cuboid}, mesh::{Mesh, Mesh3d}, pbr::{MaterialPlugin, MeshMaterial3d, StandardMaterial}, prelude::{Deref, DerefMut}, render::render_resource::{Extent3d, TextureUsages}, transform::components::Transform, utils::default, }; use bevy_egui::{ EguiGlobalSettings, EguiInputSet, EguiMultipassSchedule, EguiPlugin, input::EguiInputEvent, }; use bevy_mod_openxr::session::OxrSession; use bevy_pkv::{PersistentResourceAppExtensions, PkvStore}; use egui::{Color32, Margin, PointerButton, Pos2}; use openxr::Path; use serde::{Deserialize, Serialize}; // PUB pub use crate::apad::otdipcplugin::TabletRotation; use crate::{ MainCamera, apad::{ custom_material::{CustomMaterial, MyCustomMaterial}, keyboard::VirtualKeyboard, otdipcplugin::{OtdIpcPlugin, PenButtons, PenDelta, PenPosition}, }, egui_pages::{Overview, overview}, vrcontrollerplugin::{ LeftController, LeftControllerActions, RightController, RightControllerActions, }, vrplugin::Headset, }; #[derive(ScheduleLabel, Clone, Debug, PartialEq, Eq, Hash)] pub struct WorldspaceContextPass; #[derive(Resource, Deref)] pub struct TabletSize(Vec2); #[derive(Resource, Deref)] pub struct TabletResolutionScale(f32); #[derive(Resource, Deref, DerefMut)] struct PointerImage(Option>); #[derive(Component, Deref, DerefMut)] pub struct Keyboard(VirtualKeyboard); #[derive(Component)] pub struct Kneeboard; #[derive(Resource, Deref, DerefMut)] struct Name(String); #[derive(Resource, Deref, DerefMut)] struct KeyboardVisible(bool); #[derive(Resource, Deref, DerefMut)] struct WasPenButtonPressed(bool); #[derive(Resource, Deref, DerefMut)] struct PickedColor(Color32); #[derive(Component)] pub struct LookedAt; #[derive(Resource, Serialize, Deserialize)] pub struct KneeboardPosition { position: Vec3, rotation: Quat, } impl Default for KneeboardPosition { fn default() -> Self { Self { position: Vec3::new(0.15, -0.25, -0.40), rotation: Default::default(), } } } pub struct APadPlugin; impl Plugin for APadPlugin { fn build(&self, app: &mut App) { app.add_plugins(OtdIpcPlugin) .add_plugins(EguiPlugin::default()) .add_plugins(MaterialPlugin::::default()); app.insert_resource(PkvStore::new("Avii", "Kneeboard")) .init_persistent_resource::(); app.insert_resource(TabletSize(Vec2::new(210.0, 279.0))); app.insert_resource(TabletResolutionScale(2.25)); app.insert_resource(PointerImage(None)); app.insert_resource(Name("".to_string())); app.insert_resource(KeyboardVisible(false)); app.insert_resource(WasPenButtonPressed(false)); app.insert_resource(PickedColor(Color32::WHITE)); app.add_systems(Startup, (setup_pointer, setup_egui.after(setup_pointer))) .add_systems( Update, ( gaze, position_kneeboard, move_kneeboard, sync_camera_with_kneeboard, ), ) .add_systems(Update, pointer.in_set(EguiInputSet::InitReading)) .add_systems(WorldspaceContextPass, update); } } fn setup_pointer( mut images: ResMut>, tablet_size: Res, tablet_res: Res, mut pointer: ResMut, ) { let image = images.add({ let size = Extent3d { width: (tablet_size.x * **tablet_res) as u32, height: (tablet_size.y * **tablet_res) as u32, depth_or_array_layers: 1, }; let mut image = Image { data: Some(vec![0; (size.width * size.height * 4) as usize]), ..default() }; image.texture_descriptor.usage |= TextureUsages::RENDER_ATTACHMENT; image.texture_descriptor.size = size; image }); pointer.0 = Some(image); } fn pointer( tablet_size: Res, tablet_res: Res, ent: Single>, mesh_material: Single<&MeshMaterial3d, With>, mut pen_buttons: MessageReader, mut pen_delta: MessageReader, mut pen_position: MessageReader, mut input_writer: MessageWriter, mut materials: ResMut>, mut images: ResMut>, mut pointer: ResMut, mut pen_button_pressed: ResMut, ) { let Some(ref mut pointer) = pointer.0 else { return; }; let Some(image) = images.get_mut(&*pointer) else { return; }; image.clear(&[0u8; 4]); for pos in pen_position.read() { let x = pos.x * tablet_size.x * **tablet_res; let y = pos.y * tablet_size.y * **tablet_res; input_writer.write(EguiInputEvent { context: *ent, event: egui::Event::PointerMoved(Pos2::new(x, y)), }); for button in pen_buttons.read() { if button.a() && button.tip() { for delta in pen_delta.read() { input_writer.write(EguiInputEvent { context: *ent, event: egui::Event::MouseWheel { unit: egui::MouseWheelUnit::Line, delta: egui::Vec2::new((-delta.x) * 20., (-delta.y) * 20.), modifiers: default(), }, }); } continue; } if **pen_button_pressed != button.tip() { input_writer.write(EguiInputEvent { context: *ent, event: egui::Event::PointerButton { pos: Pos2::new(x, y), button: PointerButton::Primary, pressed: button.tip(), modifiers: Default::default(), }, }); **pen_button_pressed = button.tip(); } } draw_pointer(image, x, y); if let Some(material) = materials.get_mut(mesh_material.0.id()) { material.extension.cursor_texture = Some(pointer.clone()); } } } fn draw_pointer(image: &mut Image, x: f32, y: f32) { let x = x as i32; let y = y as i32; let distance = 3; let length = 8; // let thickness = 2; for dx in (x - length - distance - 1)..=(x - distance - 1) { let mut color = css::BLACK.into(); if dx == (x - length - distance - 1) || dx == (x - distance - 1) { color = css::WHITE.into(); } draw_pixel(image, dx, y - 1, css::WHITE.into()); draw_pixel(image, dx, y, color); draw_pixel(image, dx, y + 1, css::WHITE.into()); } for dx in (x + distance + 1)..=(x + length + distance + 1) { let mut color = css::BLACK.into(); if dx == (x + distance + 1) || dx == (x + length + distance + 1) { color = css::WHITE.into(); } draw_pixel(image, dx, y - 1, css::WHITE.into()); draw_pixel(image, dx, y, color); draw_pixel(image, dx, y + 1, css::WHITE.into()); } for dy in (y - length - distance - 1)..=(y - distance - 1) { let mut color = css::BLACK.into(); if dy == (y - length - distance - 1) || dy == (y - distance - 1) { color = css::WHITE.into(); } draw_pixel(image, x - 1, dy, css::WHITE.into()); draw_pixel(image, x, dy, color); draw_pixel(image, x + 1, dy, css::WHITE.into()); } for dy in (y + distance + 1)..=(y + length + distance + 1) { let mut color = css::BLACK.into(); if dy == (y + distance + 1) || dy == (y + length + distance + 1) { color = css::WHITE.into(); } draw_pixel(image, x - 1, dy, css::WHITE.into()); draw_pixel(image, x, dy, color); draw_pixel(image, x + 1, dy, css::WHITE.into()); } } fn draw_pixel(image: &mut Image, x: i32, y: i32, color: Color) { if x < 0 || x > image.width() as i32 || y < 0 || y > image.height() as i32 { return; } image.set_color_at(x as u32, y as u32, color).ok(); } fn setup_egui( mut commands: Commands, mut images: ResMut>, mut meshes: ResMut>, mut materials: ResMut>, mut egui_global_settings: ResMut, tablet_size: Res, tablet_res: Res, pointer: Res, ) { egui_global_settings.auto_create_primary_context = false; egui_global_settings.enable_cursor_icon_updates = false; let image = images.add({ let size = Extent3d { width: (tablet_size.x * **tablet_res) as u32, height: (tablet_size.y * **tablet_res) as u32, depth_or_array_layers: 1, }; let mut image = Image { data: Some(vec![0; (size.width * size.height * 4) as usize]), ..default() }; image.texture_descriptor.usage |= TextureUsages::RENDER_ATTACHMENT; image.texture_descriptor.size = size; image }); commands.spawn(( Camera3d::default(), RenderLayers::none(), Camera::default(), RenderTarget::Image(image.clone().into()), EguiMultipassSchedule::new(WorldspaceContextPass), )); commands.spawn(( Mesh3d(meshes.add(Cuboid::new( tablet_size.x / 1000., tablet_size.y / 1000., 0.01, ))), MeshMaterial3d(materials.add(MyCustomMaterial { base: StandardMaterial { base_color: css::WHITE.into(), unlit: true, ..Default::default() }, extension: CustomMaterial { ui_texture: Some(image.clone()), cursor_texture: pointer.clone(), }, })), Kneeboard, )); let keyboard = VirtualKeyboard::default(); commands.spawn(Keyboard(keyboard)); } fn update( mut ctx: Single<&mut bevy_egui::EguiContext>, mut input: Single<&mut bevy_egui::EguiInput>, mut keyboard: Single<&mut Keyboard>, mo: Res, ) { let bgcolor = egui::containers::Frame { fill: egui::Color32::from_rgb(43, 44, 47), inner_margin: Margin { left: 15, right: 15, top: 15, bottom: 15, }, ..Default::default() }; let focus = keyboard.is_active(ctx.get_mut()); let height = if focus { 200.0 } else { 0.0 }; egui::containers::CentralPanel::default() .frame(bgcolor) .show(ctx.get_mut(), |ui| { egui::containers::TopBottomPanel::bottom("bottom_panel") .resizable(false) .height_range(egui::Rangef::new(0., height)) .show_inside(ui, |ui| { ui.vertical_centered(|ui| { keyboard.show(ui); }); }); egui::ScrollArea::vertical().show(ui, |ui| { overview(ui, mo); // .. }); }); keyboard.bump_events(ctx.get_mut(), &mut input.0); } fn sync_camera_with_kneeboard( kneeboard: Query<&Transform, With>, mut cameras: Query<&mut Transform, (With, Without)>, ) { let Ok(kneeboard) = kneeboard.single() else { return; }; let Ok(mut camera) = cameras.single_mut() else { return; }; let mut transform = *kneeboard; transform.translation -= kneeboard.back().normalize() * 0.35; transform.rotation *= Quat::from_axis_angle(Dir3::X.normalize(), PI) * Quat::from_axis_angle(Dir3::Z.normalize(), PI); *camera = transform; } fn gaze( mut commands: Commands, position: Res, kneeboard: Single>, head: Single<&Transform, With>, ) { // let head = head; // let kneeboard = kneeboard.single().expect("a kneeboard to exist"); let facing = head.forward().normalize(); let to_target = (position.position - head.translation).normalize(); let dot = facing.dot(to_target); let cos_fov_angle: f32 = (35.0f32 / 2.0f32).to_radians().cos(); commands.entity(kneeboard.entity()).remove::(); if dot < cos_fov_angle { commands.entity(kneeboard.entity()).insert(LookedAt); } } #[allow(clippy::type_complexity)] fn position_kneeboard( mut transform: Query<(&mut Transform, Option<&LookedAt>), (With, Without)>, kneeboard: Res, head: Query<&Transform, (With, Without)>, ) { let Ok((mut transform, looked_at)) = transform.single_mut() else { return; }; let Ok(head) = head.single() else { return; }; transform.translation = kneeboard.position; transform.rotation = kneeboard.rotation; if looked_at.is_some() { transform.translation = head.translation; transform.rotation = head.rotation; } } fn move_kneeboard( session: Option>, left: Option>, right: Option>, left_transform: Query<&Transform, (With, Without)>, right_transform: Query<&Transform, (With, Without)>, mut kneeboard: ResMut, ) { let Some(session) = session else { return; }; let Some(left) = left else { return; }; let Some(right) = right else { return; }; let rot_offset: Quat = Quat::from_axis_angle(Vec3::new(1.0, 0.0, 0.0), FRAC_PI_2) * Quat::from_axis_angle(Vec3::new(0.0, 0.0, 1.0), PI); if let Ok(trigger_state) = left.squeeze_click.state(&session, Path::NULL) && trigger_state.current_state { dbg!("squeeze triggered"); let Ok(transform) = left_transform.single() else { return; }; let pos_offset = transform.up().normalize() * -0.1; kneeboard.position = transform.translation + pos_offset; kneeboard.rotation = transform.rotation * rot_offset; } if let Ok(trigger_state) = right.squeeze_click.state(&session, Path::NULL) && trigger_state.current_state { let Ok(transform) = right_transform.single() else { return; }; let pos_offset = transform.up().normalize() * -0.1; kneeboard.position = transform.translation + pos_offset; kneeboard.rotation = transform.rotation * rot_offset; } }