Skip to content

Commit

Permalink
REFAC(client): Use AudioPreprocessor
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidebeatrici
davidebeatrici committed Jul 17, 2024
1 parent 1bd56b1 commit 51bffa0
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 75 deletions.
2 changes: 1 addition & 1 deletion src/mumble/AudioConfigDialog.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,7 @@ void AudioInputDialog::showSpeexNoiseSuppressionSlider(bool show) {
void AudioInputDialog::on_Tick_timeout() {
AudioInputPtr ai = Global::get().ai;

if (!ai.get() || !ai->sppPreprocess)
if (!ai.get() || !ai->m_preprocessor)
return;

abSpeech->iBelow = qsTransmitMin->value();
Expand Down
67 changes: 24 additions & 43 deletions src/mumble/AudioInput.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,7 @@ AudioInput::AudioInput()

bEchoMulti = false;

sppPreprocess = nullptr;
sesEcho = nullptr;
sesEcho = nullptr;
srsMic = srsEcho = nullptr;

iEchoChannels = iMicChannels = 0;
Expand Down Expand Up @@ -298,8 +297,6 @@ AudioInput::~AudioInput() {
}
#endif

if (sppPreprocess)
speex_preprocess_state_destroy(sppPreprocess);
if (sesEcho)
speex_echo_state_destroy(sesEcho);

Expand Down Expand Up @@ -740,44 +737,34 @@ void AudioInput::resetAudioProcessor() {
if (!bResetProcessor)
return;

int iArg;

if (sppPreprocess)
speex_preprocess_state_destroy(sppPreprocess);
if (sesEcho)
speex_echo_state_destroy(sesEcho);

sppPreprocess = speex_preprocess_state_init(iFrameSize, iSampleRate);
m_preprocessor.init(iSampleRate, iFrameSize);
resync.reset();
selectNoiseCancel();

iArg = 1;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_VAD, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_DEREVERB, &iArg);

iArg = 30000;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_TARGET, &iArg);
m_preprocessor.toggleVAD(true);
m_preprocessor.toggleAGC(true);
m_preprocessor.toggleDereverb(true);

float v = 30000.0f / static_cast< float >(Global::get().s.iMinLoudness);
iArg = static_cast< int >(floorf(20.0f * log10f(v)));
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_MAX_GAIN, &iArg);
m_preprocessor.setAGCTarget(30000);

iArg = -60;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_DECREMENT, &iArg);
const float v = 30000.0f / static_cast< float >(Global::get().s.iMinLoudness);
m_preprocessor.setAGCMaxGain(static_cast< std::int32_t >(floorf(20.0f * log10f(v))));
m_preprocessor.setAGCDecrement(-60);

if (noiseCancel == Settings::NoiseCancelSpeex) {
iArg = Global::get().s.iSpeexNoiseCancelStrength;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &iArg);
m_preprocessor.setNoiseSuppress(Global::get().s.iSpeexNoiseCancelStrength);
}

if (iEchoChannels > 0) {
int filterSize = iFrameSize * (10 + resync.getNominalLag());
sesEcho =
speex_echo_state_init_mc(iFrameSize, filterSize, 1, bEchoMulti ? static_cast< int >(iEchoChannels) : 1);
iArg = iSampleRate;
int iArg = iSampleRate;
speex_echo_ctl(sesEcho, SPEEX_ECHO_SET_SAMPLING_RATE, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_ECHO_STATE, sesEcho);
m_preprocessor.setEchoState(sesEcho);

qWarning("AudioInput: ECHO CANCELLER ACTIVE");
} else {
Expand Down Expand Up @@ -821,24 +808,24 @@ void AudioInput::selectNoiseCancel() {
#endif
}

int iArg = 0;
bool preprocessorDenoise = false;
switch (noiseCancel) {
case Settings::NoiseCancelOff:
qWarning("AudioInput: Noise canceller disabled");
break;
case Settings::NoiseCancelSpeex:
qWarning("AudioInput: Using Speex as noise canceller");
iArg = 1;
preprocessorDenoise = true;
break;
case Settings::NoiseCancelRNN:
qWarning("AudioInput: Using ReNameNoise as noise canceller");
break;
case Settings::NoiseCancelBoth:
iArg = 1;
preprocessorDenoise = true;
qWarning("AudioInput: Using ReNameNoise and Speex as noise canceller");
break;
}
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_DENOISE, &iArg);
m_preprocessor.toggleDenoise(preprocessorDenoise);
}

int AudioInput::encodeOpusFrame(short *source, int size, EncodingOutputBuffer &buffer) {
Expand All @@ -857,7 +844,6 @@ int AudioInput::encodeOpusFrame(short *source, int size, EncodingOutputBuffer &b
}

void AudioInput::encodeAudioFrame(AudioChunk chunk) {
int iArg;
float sum;
short max;

Expand Down Expand Up @@ -897,11 +883,10 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
QMutexLocker l(&qmSpeex);
resetAudioProcessor();

speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_AGC_GAIN, &iArg);
float gainValue = static_cast< float >(iArg);
const std::int32_t gainValue = m_preprocessor.getAGCGain();

if (noiseCancel == Settings::NoiseCancelSpeex || noiseCancel == Settings::NoiseCancelBoth) {
iArg = Global::get().s.iSpeexNoiseCancelStrength - iArg;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &iArg);
m_preprocessor.setNoiseSuppress(Global::get().s.iSpeexNoiseCancelStrength - gainValue);
}

short psClean[iFrameSize];
Expand All @@ -924,7 +909,7 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
}
#endif

speex_preprocess_run(sppPreprocess, psSource);
m_preprocessor.run(*psSource);

sum = 1.0f;
for (unsigned int i = 0; i < iFrameSize; i++)
Expand All @@ -942,12 +927,10 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
static_cast< std::streamsize >(iFrameSize * sizeof(short)));
}

spx_int32_t prob = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_PROB, &prob);
fSpeechProb = static_cast< float >(prob) / 100.0f;
fSpeechProb = static_cast< float >(m_preprocessor.getSpeechProb()) / 100.0f;

// clean microphone level: peak of filtered signal attenuated by AGC gain
dPeakCleanMic = qMax(dPeakSignal - gainValue, -96.0f);
dPeakCleanMic = qMax(dPeakSignal - static_cast< float >(gainValue), -96.0f);
float level = (Global::get().s.vsVAD == Settings::SignalToNoise) ? fSpeechProb : (1.0f + dPeakCleanMic / 96.0f);

bool bIsSpeech = false;
Expand Down Expand Up @@ -1075,12 +1058,10 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
}
}

spx_int32_t increment = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
m_preprocessor.setAGCIncrement(0);
return;
} else {
spx_int32_t increment = 12;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
m_preprocessor.setAGCIncrement(12);
}

if (bIsSpeech && !bPreviousVoice) {
Expand Down
4 changes: 2 additions & 2 deletions src/mumble/AudioInput.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@
#include <vector>

#include <speex/speex_echo.h>
#include <speex/speex_preprocess.h>
#include <speex/speex_resampler.h>

#include "Audio.h"
#include "AudioOutputToken.h"
#include "AudioPreprocessor.h"
#include "EchoCancelOption.h"
#include "MumbleProtocol.h"
#include "Settings.h"
Expand Down Expand Up @@ -224,7 +224,7 @@ class AudioInput : public QThread {
static const int iFrameSize = SAMPLE_RATE / 100;

QMutex qmSpeex;
SpeexPreprocessState *sppPreprocess;
AudioPreprocessor m_preprocessor;
SpeexEchoState *sesEcho;

/// bResetEncoder is a flag that notifies
Expand Down
44 changes: 15 additions & 29 deletions src/mumble/AudioStats.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,34 +216,28 @@ void AudioNoiseWidget::paintEvent(QPaintEvent *) {
paint.fillRect(rect(), pal.color(QPalette::Window));

AudioInputPtr ai = Global::get().ai;
if (!ai.get() || !ai->sppPreprocess)
if (!ai.get() || !ai->m_preprocessor)
return;

QPolygonF poly;

ai->qmSpeex.lock();

spx_int32_t ps_size = 0;
speex_preprocess_ctl(ai->sppPreprocess, SPEEX_PREPROCESS_GET_PSD_SIZE, &ps_size);

static std::vector< spx_int32_t > noise;
noise.resize(static_cast< std::size_t >(ps_size));
static std::vector< spx_int32_t > ps;
ps.resize(static_cast< std::size_t >(ps_size));

speex_preprocess_ctl(ai->sppPreprocess, SPEEX_PREPROCESS_GET_PSD, ps.data());
speex_preprocess_ctl(ai->sppPreprocess, SPEEX_PREPROCESS_GET_NOISE_PSD, noise.data());
const std::vector< std::int32_t > ps = ai->m_preprocessor.getPSD();
const std::vector< std::int32_t > noise = ai->m_preprocessor.getNoisePSD();

ai->qmSpeex.unlock();

assert(ps.size() == noise.size());

qreal sx, sy;

sx = (static_cast< float >(width()) - 1.0f) / static_cast< float >(ps_size);
sx = (static_cast< float >(width()) - 1.0f) / static_cast< float >(ps.size());
sy = static_cast< float >(height()) - 1.0f;

poly << QPointF(0.0f, height() - 1);
float fftmul = 1.0 / (32768.0);
for (unsigned int i = 0; i < static_cast< unsigned int >(ps_size); i++) {
for (unsigned int i = 0; i < static_cast< unsigned int >(noise.size()); i++) {
qreal xp, yp;
xp = i * sx;
yp = sqrtf(sqrtf(static_cast< float >(noise[i]))) - 1.0f;
Expand All @@ -262,7 +256,7 @@ void AudioNoiseWidget::paintEvent(QPaintEvent *) {

poly.clear();

for (unsigned int i = 0; i < static_cast< unsigned int >(ps_size); i++) {
for (unsigned int i = 0; i < static_cast< unsigned int >(ps.size()); i++) {
qreal xp, yp;
xp = i * sx;
yp = sqrtf(sqrtf(static_cast< float >(ps[i]))) - 1.0f;
Expand Down Expand Up @@ -316,7 +310,7 @@ AudioStats::~AudioStats() {
void AudioStats::on_Tick_timeout() {
AudioInputPtr ai = Global::get().ai;

if (!ai.get() || !ai->sppPreprocess)
if (!ai.get() || !ai->m_preprocessor)
return;

bool nTalking = ai->isTransmitting();
Expand All @@ -332,22 +326,16 @@ void AudioStats::on_Tick_timeout() {
FORMAT_TO_TXT("%06.2f dB", ai->dPeakSignal);
qlSignalLevel->setText(txt);

spx_int32_t ps_size = 0;
speex_preprocess_ctl(ai->sppPreprocess, SPEEX_PREPROCESS_GET_PSD_SIZE, &ps_size);

static std::vector< spx_int32_t > noise;
noise.resize(static_cast< std::size_t >(ps_size));
static std::vector< spx_int32_t > ps;
ps.resize(static_cast< std::size_t >(ps_size));
const std::vector< std::int32_t > ps = ai->m_preprocessor.getPSD();
const std::vector< std::int32_t > noise = ai->m_preprocessor.getNoisePSD();

speex_preprocess_ctl(ai->sppPreprocess, SPEEX_PREPROCESS_GET_PSD, ps.data());
speex_preprocess_ctl(ai->sppPreprocess, SPEEX_PREPROCESS_GET_NOISE_PSD, noise.data());
assert(ps.size() == noise.size());

float s = 0.0f;
float n = 0.0001f;

unsigned int start = static_cast< unsigned int >(ps_size * 300) / SAMPLE_RATE;
unsigned int stop = static_cast< unsigned int >(ps_size * 2000) / SAMPLE_RATE;
unsigned int start = static_cast< unsigned int >(ps.size() * 300) / SAMPLE_RATE;
unsigned int stop = static_cast< unsigned int >(ps.size() * 2000) / SAMPLE_RATE;

for (unsigned int i = start; i < stop; i++) {
s += sqrtf(static_cast< float >(ps[i]));
Expand All @@ -357,9 +345,7 @@ void AudioStats::on_Tick_timeout() {
FORMAT_TO_TXT("%06.3f", s / n);
qlMicSNR->setText(txt);

spx_int32_t v;
speex_preprocess_ctl(ai->sppPreprocess, SPEEX_PREPROCESS_GET_AGC_GAIN, &v);
float fv = powf(10.0f, (static_cast< float >(v) / 20.0f));
float fv = powf(10.0f, (static_cast< float >(ai->m_preprocessor.getAGCGain()) / 20.0f));
FORMAT_TO_TXT("%03.0f%%", 100.0f / fv);
qlMicVolume->setText(txt);

Expand Down

0 comments on commit 51bffa0

Please sign in to comment.