Avii/guardian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

omg so much has changed idk

Browse Source
This commit is contained in:
AviiNL
2024-01-18 20:50:27 +01:00
parent f0f0d0c550
commit b0efc215ba
20 changed files with 544 additions and 1104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
crates/guardian_core/Cargo.toml
View File

@@ -35,11 +35,7 @@ thiserror = "1.0"
symspell = "0.4.3"
simsearch = "0.2"
[target.'cfg(target_os = "windows")'.dependencies.windows]
version = "0.52"
features = [
"Foundation",
"Foundation_Collections",
"Storage_Streams",
"Media_SpeechSynthesis",
]
hound = "3.5.1"
dasp = { version = "0.11.0", features = ["all"] }
urlencoding = "2.1.3"
samplerate = "0.2.4"

24
crates/guardian_core/src/dcs/mission.rs
View File

@@ -14,13 +14,13 @@ use bevy::{
};
use crossbeam_channel::{unbounded, Receiver};
use dcs_grpc::dcs::{
common::v0::{Coalition, GroupCategory::Airplane, Unit},
common::v0::{Coalition, GroupCategory, Unit},
mission::v0::{
mission_service_client::MissionServiceClient, stream_units_response::Update,
StreamUnitsRequest,
},
};
use tracing::{debug, info, warn};
use tracing::{error, info, warn};
#[derive(Clone)]
enum Response {
@@ -65,7 +65,7 @@ fn connect_to_grpc(mut commands: Commands, tokio: Res<TokioResource>, url: Res<G
.stream_units(StreamUnitsRequest {
poll_rate,
max_backoff: Some(30),
category: Airplane as i32,
category: GroupCategory::Airplane as i32,
})
.await
else {
@@ -83,7 +83,7 @@ fn connect_to_grpc(mut commands: Commands, tokio: Res<TokioResource>, url: Res<G
break;
}
Err(e) => {
debug!("Error from gRPC: {:?}", e); // verbose or debug log
error!("Error from gRPC: {:?}", e); // verbose or debug log
break;
}
}
@@ -91,6 +91,7 @@ fn connect_to_grpc(mut commands: Commands, tokio: Res<TokioResource>, url: Res<G
warn!("Disconnected from DCS: {}", url); // warn
tx.send(Response::Disconnected).ok();
tokio::time::sleep(Duration::from_secs(5)).await;
}
});
@@ -180,7 +181,22 @@ fn update_units(
}
if let Some(playername) = &event.player_name {
let playername = playername.clone();
e.insert(Player::new(playername.clone()));
if let Some(index) = playername.find('|') {
let custom_callsign = playername[..index].trim().to_string();
// testing callsign validity
if guardian_commands::call::parse_call(&format!(
"Overlord {} radio check",
custom_callsign
))
.is_ok()
{
e.insert(Callsign::new(custom_callsign.clone()));
}
}
}
if let Some(group) = &event.group {

35
crates/guardian_core/src/dcs/text.rs
View File

@@ -15,13 +15,16 @@ use bevy::{
reflect::{std_traits::ReflectDefault, Reflect},
utils::AHasher,
};
use dcs_grpc::dcs::{
common::v0::Coalition,
net::v0::{net_service_client::NetServiceClient, SendChatRequest},
use dcs_grpc::dcs::trigger::v0::{
trigger_service_client::TriggerServiceClient, OutTextForUnitRequest,
};
use tracing::info;
use crate::{components::GrpcBaseUrl, TokioResource};
use crate::{
components::{GrpcBaseUrl, Id},
TokioResource,
};
pub struct TextPlugin;
@@ -33,36 +36,38 @@ impl Plugin for TextPlugin {
fn send_text_message(
mut commands: Commands,
query: Query<(Entity, &TextMessage), Added<TextMessage>>,
query: Query<(Entity, &Id, &TextMessage), Added<TextMessage>>,
url: Res<GrpcBaseUrl>,
tokio: Res<TokioResource>,
) {
let url = url.to_string();
for (ent, id, msg) in query.iter() {
let url = url.to_string();
for (ent, msg) in query.iter() {
if msg.as_str().is_empty() {
commands.entity(ent).remove::<TextMessage>().despawn();
commands.entity(ent).remove::<TextMessage>();
continue;
}
let message = msg.clone();
let url = url.clone();
let id: u32 = id.into();
tokio.0.spawn(async move {
let Ok(mut client) = NetServiceClient::connect(url.to_string()).await else {
let Ok(mut client) = TriggerServiceClient::connect(url.to_string()).await else {
return;
};
let request = SendChatRequest {
message: message.to_string(),
coalition: Coalition::All as i32,
// target_player_id: player_id,
let request = OutTextForUnitRequest {
text: message.to_string(),
display_time: 15,
clear_view: false,
unit_id: id,
};
info!("> {}", message);
client.send_chat(request).await.ok();
client.out_text_for_unit(request).await.ok();
});
commands.entity(ent).remove::<TextMessage>().despawn();
commands.entity(ent).remove::<TextMessage>();
}
}

41
crates/guardian_core/src/srs.rs
View File

@@ -6,9 +6,8 @@ mod voice_codec;
mod voice_command;
use guardian_commands::{call::parse_call, ConsoleCommandEntered, ConsoleConfiguration};
use serde::{Deserialize, Serialize};
use simsearch::SimSearch;
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
pub use voice_command::VoiceCommand;
use crossbeam_channel::Receiver;
@@ -161,8 +160,8 @@ fn listen_srs(
commands.entity(ent).insert(ClientHandler(client_handle));
let frequency = radio.frequency;
let modulation = radio.modulation;
let id = *id;
let voice = radio.voice.clone();
let stt_url = stt_url.as_str().to_string();
radio.handle = Some(tokio.0.spawn(async move {
loop {
@@ -212,11 +211,7 @@ fn listen_srs(
wav_audio_part: None,
frequencies: vec![Frequency {
freq: frequency as f64,
modulation: if frequency <= 87_995_000 {
voice_codec::Modulation::Fm
} else {
voice_codec::Modulation::Am
},
modulation: modulation.into(),
encryption: Encryption::None,
}],
unit_id: id.into(),
@@ -266,7 +261,7 @@ fn listen_srs(
}
}
Some(data) = voice_handle.recv() => {
frames.push(synthesize(data.as_str(), &voice).await?).await;
frames.push(synthesize("192.168.0.238:10200", data.as_str()).await?).await;
}
Some(data) = voice_stream.next() => {
match data {
@@ -501,9 +496,12 @@ fn handle_voice_command(
info!("Received: {}", raw);
info!("Interpreted as: {}", cmd);
let Ok((_, call)) = parse_call(cmd) else {
// no command, casual conversations?
return;
let call = match parse_call(cmd) {
Ok((_, call)) => call,
Err(e) => {
debug!("Error parsing call: {:?}", e);
return;
}
};
let cmd = call.command;
@@ -520,21 +518,6 @@ fn handle_voice_command(
}
}
#[derive(Default, Clone, Debug, Deserialize, Serialize)]
pub struct Voice(String);
impl Voice {
pub fn new(voice: impl Into<String>) -> Self {
Self(voice.into())
}
}
impl std::fmt::Display for Voice {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
#[derive(Component)]
pub struct ClientHandler(Receiver<Client>);
@@ -542,7 +525,6 @@ pub struct ClientHandler(Receiver<Client>);
pub struct Radio {
pub frequency: u64, // the way srs wants it
pub modulation: Modulation,
pub voice: Voice,
sguid: String,
voice_sink: Option<Sender<VoiceMessage>>,
message_sink: Option<Sender<MessageRequest>>,
@@ -550,12 +532,11 @@ pub struct Radio {
}
impl Radio {
pub fn new(frequency: u64, modulation: impl Into<Modulation>, voice: Voice) -> Self {
pub fn new(frequency: u64, modulation: impl Into<Modulation>) -> Self {
let modulation = modulation.into();
Self {
frequency,
modulation,
voice,
sguid: create_sguid(),
voice_sink: None,
message_sink: None,

16
crates/guardian_core/src/srs/voice_codec.rs
View File

@@ -7,6 +7,8 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use bytes::{BufMut, BytesMut};
use tokio_util::codec::{Decoder, Encoder, LengthDelimitedCodec};
use super::message;
pub const GUID_LENGTH: usize = 22;
pub const PACKET_HEADER_LENGTH: usize = size_of::<u16>() // UInt16 Packet Length - 2 bytes
@@ -53,6 +55,20 @@ pub enum Modulation {
Disabled,
}
impl From<message::Modulation> for Modulation {
fn from(value: message::Modulation) -> Self {
match value {
super::Modulation::Am => Modulation::Am,
super::Modulation::Fm => Modulation::Fm,
super::Modulation::Intercom => Modulation::Intercom,
super::Modulation::Disabled => Modulation::Disabled,
super::Modulation::HaveQuick => Modulation::Disabled,
super::Modulation::Satcom => Modulation::Disabled,
super::Modulation::Mids => Modulation::Disabled,
}
}
}
#[derive(Debug, Clone)]
pub enum Encryption {
None,

271
crates/guardian_core/src/tts.rs
View File

@@ -1,135 +1,155 @@
// TODO: Look into SpeakNG to avoid the windows dependency https://crates.io/crates/espeakng
use std::io::Cursor;
use std::borrow::Cow;
use bevy::log;
use dasp::interpolate::sinc::Sinc;
use dasp::ring_buffer;
use dasp::signal;
use dasp::Sample;
use dasp::Signal;
use tokio::sync::Mutex;
use tracing::{debug, error, warn};
use windows::core::HSTRING;
use windows::Media::SpeechSynthesis::SpeechSynthesizer;
use windows::Storage::Streams::DataReader;
use hound::WavSpec;
use hound::WavWriter;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;
use tokio::net::ToSocketAddrs;
use crate::srs::Voice;
use serde::Deserialize;
use serde::Serialize;
#[derive(Debug)]
pub struct WinConfig {
pub voice: Option<String>,
#[derive(Debug, Deserialize)]
pub struct Packet {
#[serde(rename = "type")]
pub r#type: String,
pub version: String,
pub data_length: Option<usize>,
pub payload_length: Option<usize>,
}
impl WinConfig {
pub fn new(voice: &str) -> Self {
Self {
voice: Some(voice.to_string()),
}
}
#[derive(Serialize)]
struct RequestData {
text: String,
}
static MUTEX: Mutex<()> = Mutex::const_new(());
#[derive(Serialize)]
struct Request {
r#type: String,
data: RequestData,
}
pub async fn synthesize(text: &str, voice: &Voice) -> Result<Vec<Vec<u8>>, WinError> {
let config = WinConfig::new(&voice.to_string());
// Note, there does not seem to be a way to explicitly set 16000kHz, 16 audio bits per
// sample and mono channel.
pub async fn synthesize<A: ToSocketAddrs>(
addr: A,
text: &str,
) -> Result<Vec<Vec<u8>>, Box<dyn std::error::Error + Send + Sync + 'static>> {
let stream = TcpStream::connect(addr).await?;
// Prevent concurrent Windows TTS synthesis, as this might cause a crash.
let lock = MUTEX.lock().await;
let (mut read, mut write) = tokio::io::split(stream);
let mut voice_info = None;
if let Some(voice) = &config.voice {
let all_voices = SpeechSynthesizer::AllVoices()?;
let len = all_voices.Size()? as usize;
for i in 0..len {
let v = all_voices.GetAt(i as u32)?;
let lang = v.Language()?.to_string();
if !lang.starts_with("en-") {
continue;
}
let name = v.DisplayName()?.to_string();
if name.ends_with(voice) {
voice_info = Some(v);
break;
}
}
} else {
// default to the first english voice in the list
let all_voices = SpeechSynthesizer::AllVoices()?;
let len = all_voices.Size()? as usize;
for i in 0..len {
let v = all_voices.GetAt(i as u32)?;
let lang = v.Language()?.to_string();
if lang.starts_with("en-") {
let name = v.DisplayName()?.to_string();
debug!("Using WIN voice: {}", name);
voice_info = Some(v);
break;
}
}
if voice_info.is_none() {
error!("Could not find any english Windows TTS voice");
}
}
if voice_info.is_none() {
let all_voices = SpeechSynthesizer::AllVoices()?;
let len = all_voices.Size()? as usize;
warn!(
"Available WIN voices are (you don't have to include the `Microsoft` prefix in \
the name):"
);
for i in 0..len {
let v = all_voices.GetAt(i as u32)?;
let lang = v.Language()?.to_string();
if !lang.starts_with("en-") {
continue;
}
let name = v.DisplayName()?.to_string();
warn!("- {} ({})", name, lang);
}
}
let synth = SpeechSynthesizer::new()?;
let lang = if let Some(info) = voice_info {
synth.SetVoice(&info)?;
info.Language()?.to_string().into()
} else {
Cow::Borrowed("en")
let request = Request {
r#type: "synthesize".to_string(),
data: RequestData {
text: text.to_string(),
},
};
// the DataReader is !Send, which is why we have to process it in a local set
let stream = synth
.SynthesizeSsmlToStreamAsync(&HSTRING::from(&format!(
r#"<speak version="1.0" xml:lang="{lang}">{text}</speak>"#
)))?
.await?;
let size = stream.Size()?;
let mut request = serde_json::to_string(&request)?;
request.push('\n');
let rd = DataReader::CreateDataReader(&stream.GetInputStreamAt(0)?)?;
rd.LoadAsync(size as u32)?.await?;
write.write_all(request.as_bytes()).await?;
let mut wav = vec![0u8; size as usize];
rd.ReadBytes(wav.as_mut_slice())?;
let mut str = String::new();
let mut pcm_buffer: Vec<u8> = Vec::new();
loop {
let Ok(c) = read.read_u8().await else {
log::error!("Unable to read byte");
break;
};
drop(lock);
if c == b'\n' {
let Ok(packet) = serde_json::from_str::<Packet>(&str) else {
log::error!("Unable to read packet from: {}", str);
break;
};
str = String::new();
Ok(wav_to_opus(wav.into()).await?)
}
if let Some(length) = packet.data_length {
let mut buf = (0..length).map(|_| 0u8).collect::<Vec<_>>();
#[derive(Debug, thiserror::Error)]
pub enum WinError {
#[error("Calling WinRT API failed with error code {0}: {1}")]
Win(i32, String),
#[error("Runtime error")]
Io(#[from] std::io::Error),
#[error("failed to encode audio data as opus")]
Opus(#[from] audiopus::Error),
}
if let Err(e) = read.read_exact(&mut buf).await {
log::error!("Unable to read data: {:?}", e);
break;
}
};
impl From<windows::core::Error> for WinError {
fn from(err: windows::core::Error) -> Self {
WinError::Win(err.code().0, err.message().to_string())
if let Some(length) = packet.payload_length {
let mut buf = (0..length).map(|_| 0u8).collect::<Vec<_>>();
if let Err(e) = read.read_exact(&mut buf).await {
log::error!("Unable to read data: {:?}", e);
break;
}
pcm_buffer.append(&mut buf);
}
match packet.r#type.as_str() {
"audio-start" => {
pcm_buffer.clear();
}
"audio-stop" => {
log::info!("Audio received, start post-processing");
break;
}
_ => {}
}
}
str.push(c.into());
}
let wav = to_wav(&pcm_buffer)?;
let Ok(reader) = hound::WavReader::new(&*wav) else {
log::error!("Error opening reader");
return Err("Error opening reader".into());
};
// pcm_buffer is 22050Hz, need to convert this to 16000Hz
let samples = reader
.into_samples()
.filter_map(Result::ok)
.map(i16::to_sample::<f64>);
let signal = signal::from_interleaved_samples_iter(samples);
let ring_buffer = ring_buffer::Fixed::from([[0.0]; 100]);
let sinc = Sinc::new(ring_buffer);
let new_signal = signal.from_hz_to_hz(sinc, 22050.0, 16000.0);
let spec = WavSpec {
channels: 1,
sample_rate: 16000,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let new_wav: Vec<u8> = Vec::new();
let mut new_wav_cursor = Cursor::new(new_wav);
let mut writer = WavWriter::new(&mut new_wav_cursor, spec).unwrap();
for frame in new_signal.until_exhausted() {
writer.write_sample(frame[0].to_sample::<i16>()).unwrap();
}
writer.flush().unwrap();
drop(writer);
let wav = new_wav_cursor.into_inner();
let wav = bytes::Bytes::copy_from_slice(&wav);
Ok(wav_to_opus(wav).await?)
}
async fn wav_to_opus(wav: bytes::Bytes) -> Result<Vec<Vec<u8>>, audiopus::Error> {
@@ -160,3 +180,30 @@ async fn wav_to_opus(wav: bytes::Bytes) -> Result<Vec<Vec<u8>>, audiopus::Error>
.await
.unwrap()
}
fn to_wav(data: &[u8]) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync + 'static>> {
let input_buffer: Vec<u8> = Vec::new();
let mut input_buffer_cursor = Cursor::new(input_buffer);
let input_spec = WavSpec {
channels: 1,
sample_rate: 22050,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut input = hound::WavWriter::new(&mut input_buffer_cursor, input_spec)?;
let audio_stream = data
.chunks(2)
.map(|bytes| i16::from_le_bytes(bytes.try_into().unwrap()))
.collect::<Vec<_>>();
for frame in audio_stream {
input.write_sample(frame.to_sample::<i16>()).unwrap();
}
drop(input);
Ok(input_buffer_cursor.into_inner())
}