refactor(audio): introduce IAudioSource interface and use it

CMakeLists.txt

@@ -238,10 +238,13 @@ set(${PROJECT_NAME}_SOURCES
   src/audio/audio.h
   src/audio/backend/alsink.cpp
   src/audio/backend/alsink.h
+  src/audio/backend/alsource.cpp
+  src/audio/backend/alsource.h
   src/audio/backend/openal.cpp
   src/audio/backend/openal.h
   src/audio/iaudiosettings.h
   src/audio/iaudiosink.h
+  src/audio/iaudiosource.h
   src/chatlog/chatlinecontent.cpp
   src/chatlog/chatlinecontent.h
   src/chatlog/chatlinecontentproxy.cpp

src/audio/audio.cpp

@@ -24,34 +24,9 @@
 #endif
 #include "src/persistence/settings.h"
 
-#include <QDebug>
-
-#include <cassert>
-
 /**
  * @class Audio
  *
- * @enum Audio::Sound
- * @brief Provides the different sounds for use in the getSound function.
- * @see getSound
- *
- * @value NewMessage Returns the new message notification sound.
- * @value Test Returns the test sound.
- * @value IncomingCall Returns the incoming call sound.
- * @value OutgoingCall Returns the outgoing call sound.
- *
- * @fn QString Audio::getSound(Sound s)
- * @brief Function to get the path of the requested sound.
- *
- * @param s Name of the sound to get the path of.
- * @return The path of the requested sound.
- *
- * @fn void Audio::frameAvailable(const int16_t *pcm, size_t sample_count, uint8_t channels,
- * uint32_t sampling_rate);
- *
- * When there are input subscribers, we regularly emit captured audio frames with this signal
- * Always connect with a blocking queued connection lambda, else the behaviour is undefined
- *
  * @var Audio::AUDIO_SAMPLE_RATE
  * @brief The next best Opus would take is 24k
  *
@@ -89,37 +64,6 @@
  *
  * @note Default is 30dB; usually you don't need to alter this value.
  *
- * @fn void Audio::subscribeInput()
- * @brief Subscribe to capture sound from the opened input device.
- *
- * If the input device is not open, it will be opened before capturing.
- *
- * @fn void Audio::unsubscribeInput()
- * @brief Unsubscribe from capturing from an opened input device.
- *
- * If the input device has no more subscriptions, it will be closed.
- *
- * @fn void Audio::playMono16Sound(const QString& path)
- * @brief Play a 44100Hz mono 16bit PCM sound from a file
- *
- * @param[in] path the path to the sound file
- *
- * @fn void Audio::playMono16Sound(const QByteArray& data)
- * @brief Play a 44100Hz mono 16bit PCM sound
- *
- * @param[in] data 44100Hz mono 16bit PCM data in host byte order
- *
- * @fn void Audio::playAudioBuffer(uint sourceId, const int16_t* data, int samples,
- *                                  unsigned channels, int sampleRate)
- * @brief adds a number of audio frames to the play buffer
- *
- * @param[in] sourceId id obtained by subscribeOutput(uint &)
- * @param[in] data 16bit mono or stereo PCM data with alternating channel
- *            mapping for stereo (LRLR)
- * @param[in] samples number of samples per channel
- * @param[in] channels number of channels, currently 1 or 2 is supported
- * @param[in] sampleRate sample rate in Hertz
- *
  * @fn bool Audio::isOutputReady() const
  * @brief check if the output is ready to play audio
  *
@@ -135,24 +79,6 @@
  *
  * @return list of input devices
  *
- * @fn void Audio::subscribeOutput(uint& sid)
- * @brief register a new output source
- *
- * param[out] sid contains the sourceId if source creation was successful,
- *                unchanged otherwise
- *
- * @fn void Audio::unsubscribeOutput(uint& sid)
- * @brief unregisters an output source
- *
- * param[out] sid contains 0 if source deletion was successful,
- *                unchanged otherwise
- *
- * @fn void Audio::startLoop()
- * @brief starts looping the sound played with playMono16Sound()
- *
- * @fn void Audio::stopLoop()
- * @brief stops looping the sound played with playMono16Sound()
- *
  * @fn qreal Audio::inputGain() const
  * @brief get the current input gain
  *

src/audio/audio.h

@@ -25,6 +25,7 @@
 #include <memory>
 
 class IAudioSink;
+class IAudioSource;
 class Audio : public QObject
 {
     Q_OBJECT
@@ -60,12 +61,8 @@ public:
     virtual QStringList outDeviceNames() = 0;
     virtual QStringList inDeviceNames() = 0;
 
-    virtual void subscribeInput() = 0;
-    virtual void unsubscribeInput() = 0;
-
-    virtual void stopActive() = 0;
-
     virtual std::unique_ptr<IAudioSink> makeSink() = 0;
+    virtual std::unique_ptr<IAudioSource> makeSource() = 0;
 
 protected:
     // Public default audio settings
@@ -75,12 +72,6 @@ protected:
     static constexpr uint32_t AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL =
         AUDIO_FRAME_DURATION * AUDIO_SAMPLE_RATE / 1000;
     uint32_t AUDIO_FRAME_SAMPLE_COUNT_TOTAL = 0;
-
-signals:
-    void frameAvailable(const int16_t* pcm, size_t sample_count, uint8_t channels,
-                        uint32_t sampling_rate);
-    void volumeAvailable(float value);
-    void startActive(qreal msec);
 };
 
 #endif // AUDIO_H

src/audio/backend/alsource.cpp

@@ -0,0 +1,41 @@
+#include "src/audio/backend/alsource.h"
+#include "src/audio/backend/openal.h"
+
+/**
+ * @brief Emits audio frames captured by an input device or other audio source.
+ */
+
+/**
+ * @brief Reserves ressources for an audio source
+ * @param audio Main audio object, must have longer lifetime than this object.
+ */
+AlSource::AlSource(OpenAL& al)
+    : audio(al)
+    , killLock(QMutex::Recursive)
+{}
+
+AlSource::~AlSource()
+{
+    QMutexLocker{&killLock};
+
+    // unsubscribe only if not already killed
+    if (!killed) {
+        audio.destroySource(*this);
+        killed = true;
+    }
+}
+
+AlSource::operator bool() const
+{
+    QMutexLocker{&killLock};
+    return !killed;
+}
+
+void AlSource::kill()
+{
+    killLock.lock();
+    // this flag is only set once here, afterwards the object is considered dead
+    killed = true;
+    killLock.unlock();
+    emit invalidated();
+}

src/audio/backend/alsource.h

@@ -0,0 +1,30 @@
+#ifndef ALSOURCE_H
+#define ALSOURCE_H
+
+#include "src/audio/iaudiosource.h"
+#include <QMutex>
+#include <QObject>
+
+class OpenAL;
+class AlSource : public IAudioSource
+{
+    Q_OBJECT
+public:
+    AlSource(OpenAL& al);
+    AlSource(AlSource& src) = delete;
+    AlSource& operator=(const AlSource&) = delete;
+    AlSource(AlSource&& other) = delete;
+    AlSource& operator=(AlSource&& other) = delete;
+    ~AlSource();
+
+    operator bool() const;
+
+    void kill();
+
+private:
+    OpenAL& audio;
+    bool killed = false;
+    mutable QMutex killLock;
+};
+
+#endif // ALSOURCE_H

src/audio/backend/openal.cpp

@@ -18,8 +18,6 @@
 */
 
 #include "openal.h"
-#include "src/core/core.h"
-#include "src/core/coreav.h"
 #include "src/persistence/settings.h"
 
 #include <QDebug>
@@ -74,8 +72,9 @@ OpenAL::OpenAL()
 
     voiceTimer.setSingleShot(true);
     voiceTimer.moveToThread(audioThread);
-    connect(this, &Audio::startActive, &voiceTimer, static_cast<void (QTimer::*)(int)>(&QTimer::start));
+    connect(this, &OpenAL::startActive, &voiceTimer,
-    connect(&voiceTimer, &QTimer::timeout, this, &Audio::stopActive);
+            static_cast<void (QTimer::*)(int)>(&QTimer::start));
+    connect(&voiceTimer, &QTimer::timeout, this, &OpenAL::stopActive);
 
     connect(&captureTimer, &QTimer::timeout, this, &OpenAL::doAudio);
     captureTimer.setInterval(AUDIO_FRAME_DURATION / 2);
@@ -218,8 +217,19 @@ qreal OpenAL::maxInputThreshold() const
 void OpenAL::reinitInput(const QString& inDevDesc)
 {
     QMutexLocker locker(&audioLock);
+
+    const auto bakSources = sources;
+    sources.clear();
+
     cleanupInput();
     initInput(inDevDesc);
+
+    locker.unlock();
+    // this must happen outside `audioLock`, to avoid a deadlock when
+    // a slot on AlSource::invalidate tries to create a new source immedeately.
+    for (auto& source : bakSources) {
+        source->kill();
+    }
 }
 
 bool OpenAL::reinitOutput(const QString& outDevDesc)
@@ -308,17 +318,25 @@ void OpenAL::destroySink(AlSink& sink)
  *
  * If the input device is not open, it will be opened before capturing.
  */
-void OpenAL::subscribeInput()
+std::unique_ptr<IAudioSource> OpenAL::makeSource()
 {
     QMutexLocker locker(&audioLock);
 
     if (!autoInitInput()) {
         qWarning("Failed to subscribe to audio input device.");
-        return;
+        return {};
+    }
+
+    auto const source = new AlSource(*this);
+    if (source == nullptr) {
+        return {};
     }
 
-    ++inSubscriptions;
+    sources.insert(source);
-    qDebug() << "Subscribed to audio input device [" << inSubscriptions << "subscriptions ]";
+
+    qDebug() << "Subscribed to audio input device [" << sources.size() << "subscriptions ]";
+
+    return std::unique_ptr<IAudioSource>{source};
 }
 
 /**
@@ -326,18 +344,23 @@ void OpenAL::subscribeInput()
  *
  * If the input device has no more subscriptions, it will be closed.
  */
-void OpenAL::unsubscribeInput()
+void OpenAL::destroySource(AlSource& source)
 {
     QMutexLocker locker(&audioLock);
 
-    if (!inSubscriptions)
+    const auto s = sources.find(&source);
+    if (s == sources.end()) {
+        qWarning() << "Destroyed non-existant source";
         return;
+    }
 
-    inSubscriptions--;
+    sources.erase(s);
-    qDebug() << "Unsubscribed from audio input device [" << inSubscriptions << "subscriptions left ]";
 
-    if (!inSubscriptions)
+    qDebug() << "Unsubscribed from audio input device [" << sources.size() << "subscriptions left ]";
+
+    if (sources.empty()) {
         cleanupInput();
+    }
 }
 
 /**
@@ -646,13 +669,19 @@ void OpenAL::doInput()
         volume = 0;
     }
 
-    emit Audio::volumeAvailable(volume);
+    // NOTE(sudden6): this loop probably doesn't scale too well with many sources
+    for (auto source : sources) {
+        emit source->volumeAvailable(volume);
+    }
     if (!isActive) {
         return;
     }
 
-    emit Audio::frameAvailable(inputBuffer, AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL, channels,
+    // NOTE(sudden6): this loop probably doesn't scale too well with many sources
-                               AUDIO_SAMPLE_RATE);
+    for (auto source : sources) {
+        emit source->frameAvailable(inputBuffer, AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL, channels,
+                                    AUDIO_SAMPLE_RATE);
+    }
 }
 
 void OpenAL::doOutput()
@@ -670,7 +699,7 @@ void OpenAL::doAudio()
     // Output section does nothing
 
     // Input section
-    if (alInDev && inSubscriptions) {
+    if (alInDev && !sources.empty()) {
         doInput();
     }
 }

src/audio/backend/openal.h

@@ -23,6 +23,7 @@
 
 #include "src/audio/audio.h"
 #include "src/audio/backend/alsink.h"
+#include "src/audio/backend/alsource.h"
 
 #include <memory>
 #include <unordered_set>
@@ -89,8 +90,8 @@ public:
     std::unique_ptr<IAudioSink> makeSink();
     void destroySink(AlSink& sink);
 
-    void subscribeInput();
+    std::unique_ptr<IAudioSource> makeSource();
-    void unsubscribeInput();
+    void destroySource(AlSource& source);
 
     void startLoop(uint sourceId);
     void stopLoop(uint sourceId);
@@ -99,6 +100,8 @@ public:
 
     void playAudioBuffer(uint sourceId, const int16_t* data, int samples, unsigned channels,
                          int sampleRate);
+signals:
+    void startActive(qreal msec);
 
 protected:
     static void checkAlError() noexcept;
@@ -135,7 +138,6 @@ protected:
     QString outDev{};
 
     ALCdevice* alInDev = nullptr;
-    quint32 inSubscriptions = 0;
     QTimer captureTimer;
     QTimer cleanupTimer;
 
@@ -147,9 +149,7 @@ protected:
     // Qt containers need copy operators, so use stdlib containers
     std::unordered_set<AlSink*> sinks;
     std::unordered_set<AlSink*> soundSinks;
-
+    std::unordered_set<AlSource*> sources;
-    // number of output sources
-    int outCount = 0;
 
     int channels = 0;
     qreal gain = 0;

src/audio/backend/openal2.cpp

@@ -18,8 +18,6 @@
 */
 
 #include "openal2.h"
-#include "src/core/core.h"
-#include "src/core/coreav.h"
 #include "src/persistence/settings.h"
 
 #include <QDebug>

src/audio/iaudiosource.h

@@ -3,7 +3,14 @@
 
 #include <QObject>
 
-class Audio;
+/**
+ * @fn void Audio::frameAvailable(const int16_t *pcm, size_t sample_count, uint8_t channels,
+ * uint32_t sampling_rate);
+ *
+ * When there are input subscribers, we regularly emit captured audio frames with this signal
+ * Always connect with a blocking queued connection lambda, else the behaviour is undefined
+ */
+
 class IAudioSource : public QObject
 {
     Q_OBJECT

src/core/toxcall.cpp

@@ -31,14 +31,14 @@
 ToxCall::ToxCall(bool VideoEnabled, CoreAV& av)
     : av{&av}
     , videoEnabled{VideoEnabled}
+    , audioSource{Audio::getInstance().makeSource()}
 {}
 
 ToxCall::~ToxCall()
 {
-    Audio& audio = Audio::getInstance();
-
     QObject::disconnect(audioInConn);
-    audio.unsubscribeInput();
+    QObject::disconnect(audioSrcInvalid);
+
     if (videoEnabled) {
         QObject::disconnect(videoInConn);
         CameraSource::getInstance().unsubscribe();
@@ -105,14 +105,15 @@ ToxFriendCall::ToxFriendCall(uint32_t FriendNum, bool VideoEnabled, CoreAV& av)
     , sink(Audio::getInstance().makeSink())
     , friendId{FriendNum}
 {
-    // register audio
+    // TODO(sudden6): move this to audio source
-    Audio& audio = Audio::getInstance();
+    audioInConn =
-    audio.subscribeInput();
+        QObject::connect(audioSource.get(), &IAudioSource::frameAvailable,
-    audioInConn = QObject::connect(&Audio::getInstance(), &Audio::frameAvailable,
+                         [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
-                                   [&av, FriendNum](const int16_t* pcm, size_t samples,
+                             this->av->sendCallAudio(this->friendId, pcm, samples, chans, rate);
-                                                    uint8_t chans, uint32_t rate) {
+                         });
-                                       av.sendCallAudio(FriendNum, pcm, samples, chans, rate);
+
-                                   });
+    audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
+                                       [this]() { this->onAudioSourceInvalidated(); });
 
     if (!audioInConn) {
         qDebug() << "Audio input connection not working";
@@ -146,6 +147,20 @@ ToxFriendCall::~ToxFriendCall()
     QObject::disconnect(audioSinkInvalid);
 }
 
+void ToxFriendCall::onAudioSourceInvalidated()
+{
+    auto newSrc = Audio::getInstance().makeSource();
+    audioInConn =
+        QObject::connect(newSrc.get(), &IAudioSource::frameAvailable,
+                         [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
+                             this->av->sendCallAudio(this->friendId, pcm, samples, chans, rate);
+                         });
+    audioSource = std::move(newSrc);
+
+    audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
+                                       [this]() { this->onAudioSourceInvalidated(); });
+}
+
 void ToxFriendCall::onAudioSinkInvalidated()
 {
     auto newSink = Audio::getInstance().makeSink();
@@ -189,7 +204,7 @@ void ToxFriendCall::setState(const TOXAV_FRIEND_CALL_STATE& value)
 }
 
 void ToxFriendCall::playAudioBuffer(const int16_t* data, int samples, unsigned channels,
-                                 int sampleRate) const
+                                    int sampleRate) const
 {
     if (sink) {
         sink->playAudioBuffer(data, samples, channels, sampleRate);
@@ -201,17 +216,18 @@ ToxGroupCall::ToxGroupCall(int GroupNum, CoreAV& av)
     , groupId{GroupNum}
 {
     // register audio
-    Audio& audio = Audio::getInstance();
+    audioInConn =
-    audio.subscribeInput();
+        QObject::connect(audioSource.get(), &IAudioSource::frameAvailable,
-    audioInConn = QObject::connect(&Audio::getInstance(), &Audio::frameAvailable,
+                         [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
-                                   [&av, GroupNum](const int16_t* pcm, size_t samples,
+                             this->av->sendGroupCallAudio(this->groupId, pcm, samples, chans, rate);
-                                                   uint8_t chans, uint32_t rate) {
+                         });
-                                       av.sendGroupCallAudio(GroupNum, pcm, samples, chans, rate);
-                                   });
 
     if (!audioInConn) {
         qDebug() << "Audio input connection not working";
     }
+
+    audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
+                                       [this]() { this->onAudioSourceInvalidated(); });
 }
 
 ToxGroupCall::~ToxGroupCall()
@@ -220,6 +236,23 @@ ToxGroupCall::~ToxGroupCall()
     clearPeers();
 }
 
+void ToxGroupCall::onAudioSourceInvalidated()
+{
+    auto newSrc = Audio::getInstance().makeSource();
+    // TODO(sudden6): move this to audio source
+    audioInConn =
+        QObject::connect(audioSource.get(), &IAudioSource::frameAvailable,
+                         [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
+                             this->av->sendGroupCallAudio(this->groupId, pcm, samples, chans, rate);
+                         });
+
+    audioSource = std::move(newSrc);
+
+    audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
+                                       [this]() { this->onAudioSourceInvalidated(); });
+}
+
+
 void ToxGroupCall::onAudioSinkInvalidated(ToxPk peerId)
 {
     removePeer(peerId);

src/core/toxcall.h

@@ -2,6 +2,7 @@
 #define TOXCALL_H
 
 #include "src/audio/iaudiosink.h"
+#include "src/audio/iaudiosource.h"
 #include <src/core/toxpk.h>
 #include <tox/toxav.h>
 
@@ -60,6 +61,8 @@ protected:
     QMetaObject::Connection videoInConn;
     bool videoEnabled{false};
     bool nullVideoBitrate{false};
+    std::unique_ptr<IAudioSource> audioSource = nullptr;
+    QMetaObject::Connection audioSrcInvalid;
 };
 
 class ToxFriendCall : public ToxCall
@@ -77,8 +80,7 @@ public:
     TOXAV_FRIEND_CALL_STATE getState() const;
     void setState(const TOXAV_FRIEND_CALL_STATE& value);
 
-    void playAudioBuffer(const int16_t* data, int samples, unsigned channels,
+    void playAudioBuffer(const int16_t* data, int samples, unsigned channels, int sampleRate) const;
-                                     int sampleRate) const;
 
 protected:
     std::unique_ptr<QTimer> timeoutTimer;
@@ -106,7 +108,8 @@ public:
     ToxGroupCall& operator=(ToxGroupCall&& other) = delete;
     void removePeer(ToxPk peerId);
 
-    void playAudioBuffer(const ToxPk& peer, const int16_t* data, int samples, unsigned channels, int sampleRate);
+    void playAudioBuffer(const ToxPk& peer, const int16_t* data, int samples, unsigned channels,
+                         int sampleRate);
 
 private:
     void addPeer(ToxPk peerId);

src/widget/form/settings/avform.cpp

@@ -29,6 +29,7 @@
 
 #include "src/audio/audio.h"
 #include "src/audio/iaudiosettings.h"
+#include "src/audio/iaudiosource.h"
 #include "src/core/core.h"
 #include "src/core/coreav.h"
 #include "src/video/cameradevice.h"
@@ -50,7 +51,6 @@ AVForm::AVForm(Audio* audio, CoreAV* coreAV, CameraSource& camera, IAudioSetting
     , coreAV{coreAV}
     , audioSettings{audioSettings}
     , videoSettings{videoSettings}
-    , subscribedToAudioIn(false)
     , camVideoSurface(nullptr)
     , camera(camera)
 {
@@ -98,7 +98,6 @@ AVForm::AVForm(Audio* audio, CoreAV* coreAV, CameraSource& camera, IAudioSetting
     audioThresholdSlider->setTracking(false);
     audioThresholdSlider->installEventFilter(this);
 
-    connect(audio, &Audio::volumeAvailable, this, &AVForm::setVolume);
     volumeDisplay->setMaximum(totalSliderSteps);
 
     fillAudioQualityComboBox();
@@ -120,13 +119,8 @@ AVForm::~AVForm()
 
 void AVForm::hideEvent(QHideEvent* event)
 {
-    if (subscribedToAudioIn) {
-        // TODO: This should not be done in show/hide events
-        audio->unsubscribeInput();
-        subscribedToAudioIn = false;
-    }
-
     audioSink.reset();
+    audioSrc.reset();
 
     if (camVideoSurface) {
         camVideoSurface->setSource(nullptr);
@@ -144,10 +138,9 @@ void AVForm::showEvent(QShowEvent* event)
     createVideoSurface();
     getVideoDevices();
 
-    if (!subscribedToAudioIn) {
+    if (audioSrc == nullptr) {
-        // TODO: This should not be done in show/hide events
+        audioSrc = audio->makeSource();
-        audio->subscribeInput();
+        connect(audioSrc.get(), &IAudioSource::volumeAvailable, this, &AVForm::setVolume);
-        subscribedToAudioIn = true;
     }
 
     if (audioSink == nullptr) {
@@ -540,17 +533,17 @@ void AVForm::on_inDevCombobox_currentIndexChanged(int deviceIndex)
     const bool inputEnabled = deviceIndex > 0;
     audioSettings->setAudioInDevEnabled(inputEnabled);
 
-    QString deviceName;
+    QString deviceName{};
     if (inputEnabled) {
         deviceName = inDevCombobox->itemText(deviceIndex);
     }
 
-    audioSettings->setInDev(deviceName);
+    const QString oldName = audioSettings->getInDev();
-
+    if (oldName != deviceName) {
-    audio->reinitInput(deviceName);
+        audioSettings->setInDev(deviceName);
-    subscribedToAudioIn = inputEnabled;
+        audio->reinitInput(deviceName);
-    if (inputEnabled) {
+        audioSrc = audio->makeSource();
-        audio->subscribeInput();
+        connect(audioSrc.get(), &IAudioSource::volumeAvailable, this, &AVForm::setVolume);
     }
 
     microphoneSlider->setEnabled(inputEnabled);

src/widget/form/settings/avform.h

@@ -31,12 +31,13 @@
 #include <memory>
 
 class Audio;
+class IAudioSettings;
+class IAudioSink;
+class IAudioSource;
 class CameraSource;
 class CoreAV;
-class IAudioSettings;
 class IVideoSettings;
 class VideoSurface;
-class IAudioSink;
 class AVForm : public GenericForm, private Ui::AVForm
 {
     Q_OBJECT
@@ -103,6 +104,7 @@ private:
 
     bool subscribedToAudioIn;
     std::unique_ptr<IAudioSink> audioSink = nullptr;
+    std::unique_ptr<IAudioSource> audioSrc = nullptr;
     VideoSurface* camVideoSurface;
     CameraSource& camera;
     QVector<QPair<QString, QString>> videoDeviceList;