From 1730f390af533808f4c1d7898e6845464092afe3 Mon Sep 17 00:00:00 2001 From: Davide Beatrici Date: Sun, 2 Jun 2024 00:35:15 +0200 Subject: [PATCH] FEAT(client, server): Implement loopback while still sending to others --- src/MumbleProtocol.h | 5 +- src/mumble/AudioConfigDialog.cpp | 10 +- src/mumble/AudioInput.cpp | 37 +++-- src/mumble/AudioWizard.cpp | 2 +- src/mumble/EnumStringConversions.cpp | 10 +- src/mumble/Settings.h | 4 +- src/murmur/Server.cpp | 194 ++++++++++++++------------- 7 files changed, 149 insertions(+), 113 deletions(-) diff --git a/src/MumbleProtocol.h b/src/MumbleProtocol.h index 5732d6a5c3b..fd95d5aeae5 100644 --- a/src/MumbleProtocol.h +++ b/src/MumbleProtocol.h @@ -89,8 +89,9 @@ namespace Protocol { }; namespace ReservedTargetIDs { - constexpr unsigned int REGULAR_SPEECH = 0; - constexpr unsigned int SERVER_LOOPBACK = 31; + constexpr unsigned int REGULAR_SPEECH = 0; + constexpr unsigned int SERVER_LOOPBACK_REGULAR = 30; + constexpr unsigned int SERVER_LOOPBACK_ONLY = 31; } // namespace ReservedTargetIDs using audio_context_t = byte; diff --git a/src/mumble/AudioConfigDialog.cpp b/src/mumble/AudioConfigDialog.cpp index cf83d6fa812..8918c52763b 100644 --- a/src/mumble/AudioConfigDialog.cpp +++ b/src/mumble/AudioConfigDialog.cpp @@ -640,8 +640,10 @@ AudioOutputDialog::AudioOutputDialog(Settings &st) : ConfigWidget(st) { qcbSystem->setEnabled(qcbSystem->count() > 1); qcbLoopback->addItem(tr("None"), Settings::None); - qcbLoopback->addItem(tr("Local"), Settings::Local); - qcbLoopback->addItem(tr("Server"), Settings::Server); + qcbLoopback->addItem(tr("Local (don't send to others)"), Settings::LocalOnly); + qcbLoopback->addItem(tr("Local (send to others)"), Settings::LocalRegular); + qcbLoopback->addItem(tr("Server (don't send to others)"), Settings::ServerOnly); + qcbLoopback->addItem(tr("Server (send to others)"), Settings::ServerRegular); qcbDevice->view()->setTextElideMode(Qt::ElideRight); @@ -729,7 +731,7 @@ void AudioOutputDialog::load(const Settings &r) { enablePulseAudioAttenuationOptionsFor(AudioOutputRegistrar::current); loadSlider(qsJitter, r.iJitterBufferSize); - loadComboBox(qcbLoopback, r.lmLoopMode); + loadComboBox(qcbLoopback, QVariant::fromValue(r.lmLoopMode)); loadSlider(qsPacketDelay, static_cast< int >(r.dMaxPacketDelay)); loadSlider(qsPacketLoss, static_cast< int >(r.dPacketLoss * 100.0f + 0.5f)); qsbMinimumDistance->setValue(r.fAudioMinDistance); @@ -755,7 +757,7 @@ void AudioOutputDialog::save() const { s.bAttenuateUsersOnPrioritySpeak = qcbAttenuateUsersOnPrioritySpeak->isChecked(); s.iJitterBufferSize = qsJitter->value(); s.qsAudioOutput = qcbSystem->currentText(); - s.lmLoopMode = static_cast< Settings::LoopMode >(qcbLoopback->currentIndex()); + s.lmLoopMode = qcbLoopback->currentData().value< Settings::LoopMode >(); s.dMaxPacketDelay = static_cast< float >(qsPacketDelay->value()); s.dPacketLoss = static_cast< float >(qsPacketLoss->value()) / 100.0f; s.fAudioMinDistance = static_cast< float >(qsbMinimumDistance->value()); diff --git a/src/mumble/AudioInput.cpp b/src/mumble/AudioInput.cpp index 3187704d1b8..2c9388f2993 100644 --- a/src/mumble/AudioInput.cpp +++ b/src/mumble/AudioInput.cpp @@ -992,7 +992,8 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) { ClientUser *p = ClientUser::get(Global::get().uiSession); bool bTalkingWhenMuted = false; - if (Global::get().s.bMute || ((Global::get().s.lmLoopMode != Settings::Local) && p && (p->bMute || p->bSuppress)) + if (Global::get().s.bMute + || ((Global::get().s.lmLoopMode != Settings::LocalOnly) && p && (p->bMute || p->bSuppress)) || Global::get().bPushToMute || (voiceTargetID < 0)) { bTalkingWhenMuted = bIsSpeech; bIsSpeech = false; @@ -1173,8 +1174,16 @@ void AudioInput::flushCheck(const QByteArray &frame, bool terminator, std::int32 // accordingly once the client whispers for the next time. Global::get().iPrevTarget = 0; } - if (Global::get().s.lmLoopMode == Settings::Server) { - audioData.targetOrContext = Mumble::Protocol::ReservedTargetIDs::SERVER_LOOPBACK; + + switch (Global::get().s.lmLoopMode) { + case Settings::ServerOnly: + audioData.targetOrContext = Mumble::Protocol::ReservedTargetIDs::SERVER_LOOPBACK_ONLY; + break; + case Settings::ServerRegular: + audioData.targetOrContext = Mumble::Protocol::ReservedTargetIDs::SERVER_LOOPBACK_REGULAR; + break; + default: + break; } audioData.usedCodec = m_codec; @@ -1215,14 +1224,20 @@ void AudioInput::flushCheck(const QByteArray &frame, bool terminator, std::int32 } } - if (Global::get().s.lmLoopMode == Settings::Local) { - // Only add audio data to local loop buffer - LoopUser::lpLoopy.addFrame(audioData); - } else { - // Encode audio frame and send out - gsl::span< const Mumble::Protocol::byte > encodedAudioPacket = m_udpEncoder.encodeAudioPacket(audioData); - - sendAudioFrame(encodedAudioPacket); + switch (Global::get().s.lmLoopMode) { + case Settings::LocalOnly: + // Only add audio data to local loop buffer + LoopUser::lpLoopy.addFrame(audioData); + break; + case Settings::LocalRegular: + LoopUser::lpLoopy.addFrame(audioData); + [[fallthrough]]; + default: { + // Encode audio frame and send out + gsl::span< const Mumble::Protocol::byte > encodedAudioPacket = m_udpEncoder.encodeAudioPacket(audioData); + + sendAudioFrame(encodedAudioPacket); + } } qlFrames.clear(); diff --git a/src/mumble/AudioWizard.cpp b/src/mumble/AudioWizard.cpp index 608b1c55f53..d8aa944e3f7 100644 --- a/src/mumble/AudioWizard.cpp +++ b/src/mumble/AudioWizard.cpp @@ -173,7 +173,7 @@ AudioWizard::AudioWizard(QWidget *p) : QWizard(p) { updateTriggerWidgets(qrPTT->isChecked()); sOldSettings = Global::get().s; - Global::get().s.lmLoopMode = Settings::Local; + Global::get().s.lmLoopMode = Settings::LocalOnly; Global::get().s.dPacketLoss = 0.0; Global::get().s.dMaxPacketDelay = 0.0; Global::get().s.bMute = true; diff --git a/src/mumble/EnumStringConversions.cpp b/src/mumble/EnumStringConversions.cpp index 48e3b8b1fec..c0c442438d9 100644 --- a/src/mumble/EnumStringConversions.cpp +++ b/src/mumble/EnumStringConversions.cpp @@ -14,10 +14,12 @@ PROCESS(Settings::VADSource, Amplitude, "Amplitude") \ PROCESS(Settings::VADSource, SignalToNoise, "SignalToNoise") -#define LOOP_MODE_VALUES \ - PROCESS(Settings::LoopMode, None, "None") \ - PROCESS(Settings::LoopMode, Local, "Local") \ - PROCESS(Settings::LoopMode, Server, "Server") +#define LOOP_MODE_VALUES \ + PROCESS(Settings::LoopMode, None, "None") \ + PROCESS(Settings::LoopMode, LocalOnly, "LocalOnly") \ + PROCESS(Settings::LoopMode, ServerOnly, "ServerOnly") \ + PROCESS(Settings::LoopMode, LocalRegular, "LocalRegular") \ + PROCESS(Settings::LoopMode, ServerRegular, "ServerRegular") #define CHANNEL_EXPAND_VALUES \ PROCESS(Settings::ChannelExpand, NoChannels, "NoChannels") \ diff --git a/src/mumble/Settings.h b/src/mumble/Settings.h index 09b37d69388..13ac86fe206 100644 --- a/src/mumble/Settings.h +++ b/src/mumble/Settings.h @@ -188,7 +188,7 @@ struct OverlaySettings { struct Settings { enum AudioTransmit { Continuous, VAD, PushToTalk }; enum VADSource { Amplitude, SignalToNoise }; - enum LoopMode { None, Local, Server }; + enum LoopMode { None, LocalOnly, ServerOnly, LocalRegular, ServerRegular }; enum ChannelExpand { NoChannels, ChannelsWithUsers, AllChannels }; enum ChannelDrag { Ask, DoNothing, Move }; enum ServerShow { ShowPopulated, ShowReachable, ShowAll }; @@ -579,4 +579,6 @@ struct Settings { QString findSettingsLocation(bool legacy = false, bool *foundExistingFile = nullptr) const; }; +Q_DECLARE_METATYPE(Settings::LoopMode) + #endif // MUMBLE_MUMBLE_SETTINGS_H_ diff --git a/src/murmur/Server.cpp b/src/murmur/Server.cpp index bc89c653066..74fcb50fefa 100644 --- a/src/murmur/Server.cpp +++ b/src/murmur/Server.cpp @@ -1171,114 +1171,128 @@ void Server::processMsg(ServerUser *u, Mumble::Protocol::AudioData audioData, Au buffer.clear(); - if (audioData.targetOrContext == Mumble::Protocol::ReservedTargetIDs::SERVER_LOOPBACK) { - buffer.forceAddReceiver(*u, Mumble::Protocol::AudioContext::NORMAL, audioData.containsPositionalData); - } else if (audioData.targetOrContext == Mumble::Protocol::ReservedTargetIDs::REGULAR_SPEECH) { - Channel *c = u->cChannel; - - // Send audio to all users that are listening to the channel - foreach (unsigned int currentSession, m_channelListenerManager.getListenersForChannel(c->iId)) { - ServerUser *pDst = static_cast< ServerUser * >(qhUsers.value(currentSession)); - if (pDst) { - buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::LISTEN, audioData.containsPositionalData, - m_channelListenerManager.getListenerVolumeAdjustment(pDst->uiSession, c->iId)); + switch (audioData.targetOrContext) { + case Mumble::Protocol::ReservedTargetIDs::SERVER_LOOPBACK_ONLY: + buffer.forceAddReceiver(*u, Mumble::Protocol::AudioContext::NORMAL, audioData.containsPositionalData); + break; + case Mumble::Protocol::ReservedTargetIDs::SERVER_LOOPBACK_REGULAR: + buffer.forceAddReceiver(*u, Mumble::Protocol::AudioContext::NORMAL, audioData.containsPositionalData); + [[fallthrough]]; + case Mumble::Protocol::ReservedTargetIDs::REGULAR_SPEECH: { + Channel *c = u->cChannel; + + // Send audio to all users that are listening to the channel + foreach (unsigned int currentSession, m_channelListenerManager.getListenersForChannel(c->iId)) { + ServerUser *pDst = static_cast< ServerUser * >(qhUsers.value(currentSession)); + if (pDst) { + buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::LISTEN, + audioData.containsPositionalData, + m_channelListenerManager.getListenerVolumeAdjustment(pDst->uiSession, c->iId)); + } } - } - // Send audio to all users in the same channel - for (User *p : c->qlUsers) { - ServerUser *pDst = static_cast< ServerUser * >(p); + // Send audio to all users in the same channel + for (User *p : c->qlUsers) { + ServerUser *pDst = static_cast< ServerUser * >(p); - buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::NORMAL, audioData.containsPositionalData); - } + buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::NORMAL, audioData.containsPositionalData); + } + + // Send audio to all linked channels the user has speak-permission + if (!c->qhLinks.isEmpty()) { + QSet< Channel * > chans = c->allLinks(); + chans.remove(c); - // Send audio to all linked channels the user has speak-permission - if (!c->qhLinks.isEmpty()) { - QSet< Channel * > chans = c->allLinks(); - chans.remove(c); - - QMutexLocker qml(&qmCache); - - for (Channel *l : chans) { - if (ChanACL::hasPermission(u, l, ChanACL::Speak, &acCache)) { - // Send the audio stream to all users that are listening to the linked channel - for (unsigned int currentSession : m_channelListenerManager.getListenersForChannel(l->iId)) { - ServerUser *pDst = static_cast< ServerUser * >(qhUsers.value(currentSession)); - if (pDst) { - buffer.addReceiver( - *u, *pDst, Mumble::Protocol::AudioContext::LISTEN, audioData.containsPositionalData, - m_channelListenerManager.getListenerVolumeAdjustment(pDst->uiSession, l->iId)); + QMutexLocker qml(&qmCache); + + for (Channel *l : chans) { + if (ChanACL::hasPermission(u, l, ChanACL::Speak, &acCache)) { + // Send the audio stream to all users that are listening to the linked channel + for (unsigned int currentSession : m_channelListenerManager.getListenersForChannel(l->iId)) { + ServerUser *pDst = static_cast< ServerUser * >(qhUsers.value(currentSession)); + if (pDst) { + buffer.addReceiver( + *u, *pDst, Mumble::Protocol::AudioContext::LISTEN, audioData.containsPositionalData, + m_channelListenerManager.getListenerVolumeAdjustment(pDst->uiSession, l->iId)); + } } - } - // Send audio to users in the linked channel - for (User *p : l->qlUsers) { - ServerUser *pDst = static_cast< ServerUser * >(p); + // Send audio to users in the linked channel + for (User *p : l->qlUsers) { + ServerUser *pDst = static_cast< ServerUser * >(p); - buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::NORMAL, - audioData.containsPositionalData); + buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::NORMAL, + audioData.containsPositionalData); + } } } } + + break; } - } else if (u->qmTargets.contains(static_cast< int >(audioData.targetOrContext))) { // Whisper/Shout - QSet< ServerUser * > channel; - QSet< ServerUser * > direct; - QHash< ServerUser *, VolumeAdjustment > cachedListeners; - - if (u->qmTargetCache.contains(static_cast< int >(audioData.targetOrContext))) { - ZoneScopedN(TracyConstants::AUDIO_WHISPER_CACHE_STORE); - - const WhisperTargetCache &cache = u->qmTargetCache.value(static_cast< int >(audioData.targetOrContext)); - channel = cache.channelTargets; - direct = cache.directTargets; - cachedListeners = cache.listeningTargets; - } else { - ZoneScopedN(TracyConstants::AUDIO_WHISPER_CACHE_CREATE); + default: + if (u->qmTargets.contains(static_cast< int >(audioData.targetOrContext))) { // Whisper/Shout + QSet< ServerUser * > channel; + QSet< ServerUser * > direct; + QHash< ServerUser *, VolumeAdjustment > cachedListeners; - const unsigned int uiSession = u->uiSession; - qrwlVoiceThread.unlock(); - qrwlVoiceThread.lockForWrite(); + if (u->qmTargetCache.contains(static_cast< int >(audioData.targetOrContext))) { + ZoneScopedN(TracyConstants::AUDIO_WHISPER_CACHE_STORE); - if (!qhUsers.contains(uiSession)) { - return; - } + const WhisperTargetCache &cache = + u->qmTargetCache.value(static_cast< int >(audioData.targetOrContext)); + channel = cache.channelTargets; + direct = cache.directTargets; + cachedListeners = cache.listeningTargets; + } else { + ZoneScopedN(TracyConstants::AUDIO_WHISPER_CACHE_CREATE); - // Create cache entry for the given target - // Note: We have to compute the cache entry and add it to the user's cache store in an atomic - // transaction (ensured by the lock) to avoid running into situations in which a user from the cache - // gets deleted without this particular cache entry being purged (which happens, if the cache entry is - // in the store at the point of deleting the user). - const WhisperTarget &wt = u->qmTargets.value(static_cast< int >(audioData.targetOrContext)); - WhisperTargetCache cache = createWhisperTargetCacheFor(*u, wt); + const unsigned int uiSession = u->uiSession; + qrwlVoiceThread.unlock(); + qrwlVoiceThread.lockForWrite(); - u->qmTargetCache.insert(static_cast< int >(audioData.targetOrContext), std::move(cache)); + if (!qhUsers.contains(uiSession)) { + return; + } + // Create cache entry for the given target + // Note: We have to compute the cache entry and add it to the user's cache store in an atomic + // transaction (ensured by the lock) to avoid running into situations in which a user from the cache + // gets deleted without this particular cache entry being purged (which happens, if the cache entry + // is in the store at the point of deleting the user). + const WhisperTarget &wt = u->qmTargets.value(static_cast< int >(audioData.targetOrContext)); + WhisperTargetCache cache = createWhisperTargetCacheFor(*u, wt); - qrwlVoiceThread.unlock(); - qrwlVoiceThread.lockForRead(); - if (!qhUsers.contains(uiSession)) - return; - } + u->qmTargetCache.insert(static_cast< int >(audioData.targetOrContext), std::move(cache)); - // These users receive the audio because someone is shouting to their channel - for (ServerUser *pDst : channel) { - buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::SHOUT, audioData.containsPositionalData); - } - // These users receive audio because someone is whispering to them - for (ServerUser *pDst : direct) { - buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::WHISPER, audioData.containsPositionalData); - } - // These users receive audio because someone is sending audio to one of their listeners - QHashIterator< ServerUser *, VolumeAdjustment > it(cachedListeners); - while (it.hasNext()) { - it.next(); - ServerUser *user = it.key(); - const VolumeAdjustment &volumeAdjustment = it.value(); - - buffer.addReceiver(*u, *user, Mumble::Protocol::AudioContext::LISTEN, audioData.containsPositionalData, - volumeAdjustment); - } + + qrwlVoiceThread.unlock(); + qrwlVoiceThread.lockForRead(); + if (!qhUsers.contains(uiSession)) + return; + } + + // These users receive the audio because someone is shouting to their channel + for (ServerUser *pDst : channel) { + buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::SHOUT, + audioData.containsPositionalData); + } + // These users receive audio because someone is whispering to them + for (ServerUser *pDst : direct) { + buffer.addReceiver(*u, *pDst, Mumble::Protocol::AudioContext::WHISPER, + audioData.containsPositionalData); + } + // These users receive audio because someone is sending audio to one of their listeners + QHashIterator< ServerUser *, VolumeAdjustment > it(cachedListeners); + while (it.hasNext()) { + it.next(); + ServerUser *user = it.key(); + const VolumeAdjustment &volumeAdjustment = it.value(); + + buffer.addReceiver(*u, *user, Mumble::Protocol::AudioContext::LISTEN, + audioData.containsPositionalData, volumeAdjustment); + } + } } ZoneNamedN(__tracy_scoped_zone2, TracyConstants::AUDIO_SENDOUT_ZONE, true);