michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this file, michael@0: * You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: #include "MediaEngineWebRTC.h" michael@0: #include michael@0: #include michael@0: #include "mozilla/Assertions.h" michael@0: #include "MediaTrackConstraints.h" michael@0: michael@0: // scoped_ptr.h uses FF michael@0: #ifdef FF michael@0: #undef FF michael@0: #endif michael@0: #include "webrtc/modules/audio_device/opensl/single_rw_fifo.h" michael@0: michael@0: #define CHANNELS 1 michael@0: #define ENCODING "L16" michael@0: #define DEFAULT_PORT 5555 michael@0: michael@0: #define SAMPLE_RATE 256000 michael@0: #define SAMPLE_FREQUENCY 16000 michael@0: #define SAMPLE_LENGTH ((SAMPLE_FREQUENCY*10)/1000) michael@0: michael@0: // These are restrictions from the webrtc.org code michael@0: #define MAX_CHANNELS 2 michael@0: #define MAX_SAMPLING_FREQ 48000 // Hz - multiple of 100 michael@0: michael@0: #define MAX_AEC_FIFO_DEPTH 200 // ms - multiple of 10 michael@0: static_assert(!(MAX_AEC_FIFO_DEPTH % 10), "Invalid MAX_AEC_FIFO_DEPTH"); michael@0: michael@0: namespace mozilla { michael@0: michael@0: #ifdef LOG michael@0: #undef LOG michael@0: #endif michael@0: michael@0: #ifdef PR_LOGGING michael@0: extern PRLogModuleInfo* GetMediaManagerLog(); michael@0: #define LOG(msg) PR_LOG(GetMediaManagerLog(), PR_LOG_DEBUG, msg) michael@0: #else michael@0: #define LOG(msg) michael@0: #endif michael@0: michael@0: /** michael@0: * Webrtc audio source. michael@0: */ michael@0: NS_IMPL_ISUPPORTS0(MediaEngineWebRTCAudioSource) michael@0: michael@0: // XXX temp until MSG supports registration michael@0: StaticAutoPtr gFarendObserver; michael@0: michael@0: AudioOutputObserver::AudioOutputObserver() michael@0: : mPlayoutFreq(0) michael@0: , mPlayoutChannels(0) michael@0: , mChunkSize(0) michael@0: , mSamplesSaved(0) michael@0: { michael@0: // Buffers of 10ms chunks michael@0: mPlayoutFifo = new webrtc::SingleRwFifo(MAX_AEC_FIFO_DEPTH/10); michael@0: } michael@0: michael@0: AudioOutputObserver::~AudioOutputObserver() michael@0: { michael@0: } michael@0: michael@0: void michael@0: AudioOutputObserver::Clear() michael@0: { michael@0: while (mPlayoutFifo->size() > 0) { michael@0: (void) mPlayoutFifo->Pop(); michael@0: } michael@0: } michael@0: michael@0: FarEndAudioChunk * michael@0: AudioOutputObserver::Pop() michael@0: { michael@0: return (FarEndAudioChunk *) mPlayoutFifo->Pop(); michael@0: } michael@0: michael@0: uint32_t michael@0: AudioOutputObserver::Size() michael@0: { michael@0: return mPlayoutFifo->size(); michael@0: } michael@0: michael@0: // static michael@0: void michael@0: AudioOutputObserver::InsertFarEnd(const AudioDataValue *aBuffer, uint32_t aSamples, bool aOverran, michael@0: int aFreq, int aChannels, AudioSampleFormat aFormat) michael@0: { michael@0: if (mPlayoutChannels != 0) { michael@0: if (mPlayoutChannels != static_cast(aChannels)) { michael@0: MOZ_CRASH(); michael@0: } michael@0: } else { michael@0: MOZ_ASSERT(aChannels <= MAX_CHANNELS); michael@0: mPlayoutChannels = static_cast(aChannels); michael@0: } michael@0: if (mPlayoutFreq != 0) { michael@0: if (mPlayoutFreq != static_cast(aFreq)) { michael@0: MOZ_CRASH(); michael@0: } michael@0: } else { michael@0: MOZ_ASSERT(aFreq <= MAX_SAMPLING_FREQ); michael@0: MOZ_ASSERT(!(aFreq % 100), "Sampling rate for far end data should be multiple of 100."); michael@0: mPlayoutFreq = aFreq; michael@0: mChunkSize = aFreq/100; // 10ms michael@0: } michael@0: michael@0: #ifdef LOG_FAREND_INSERTION michael@0: static FILE *fp = fopen("insertfarend.pcm","wb"); michael@0: #endif michael@0: michael@0: if (mSaved) { michael@0: // flag overrun as soon as possible, and only once michael@0: mSaved->mOverrun = aOverran; michael@0: aOverran = false; michael@0: } michael@0: // Rechunk to 10ms. michael@0: // The AnalyzeReverseStream() and WebRtcAec_BufferFarend() functions insist on 10ms michael@0: // samples per call. Annoying... michael@0: while (aSamples) { michael@0: if (!mSaved) { michael@0: mSaved = (FarEndAudioChunk *) moz_xmalloc(sizeof(FarEndAudioChunk) + michael@0: (mChunkSize * aChannels - 1)*sizeof(int16_t)); michael@0: mSaved->mSamples = mChunkSize; michael@0: mSaved->mOverrun = aOverran; michael@0: aOverran = false; michael@0: } michael@0: uint32_t to_copy = mChunkSize - mSamplesSaved; michael@0: if (to_copy > aSamples) { michael@0: to_copy = aSamples; michael@0: } michael@0: michael@0: int16_t *dest = &(mSaved->mData[mSamplesSaved * aChannels]); michael@0: ConvertAudioSamples(aBuffer, dest, to_copy * aChannels); michael@0: michael@0: #ifdef LOG_FAREND_INSERTION michael@0: if (fp) { michael@0: fwrite(&(mSaved->mData[mSamplesSaved * aChannels]), to_copy * aChannels, sizeof(int16_t), fp); michael@0: } michael@0: #endif michael@0: aSamples -= to_copy; michael@0: mSamplesSaved += to_copy; michael@0: aBuffer += to_copy * aChannels; michael@0: michael@0: if (mSamplesSaved >= mChunkSize) { michael@0: int free_slots = mPlayoutFifo->capacity() - mPlayoutFifo->size(); michael@0: if (free_slots <= 0) { michael@0: // XXX We should flag an overrun for the reader. We can't drop data from it due to michael@0: // thread safety issues. michael@0: break; michael@0: } else { michael@0: mPlayoutFifo->Push((int8_t *) mSaved.forget()); // takes ownership michael@0: mSamplesSaved = 0; michael@0: } michael@0: } michael@0: } michael@0: } michael@0: michael@0: void michael@0: MediaEngineWebRTCAudioSource::GetName(nsAString& aName) michael@0: { michael@0: if (mInitDone) { michael@0: aName.Assign(mDeviceName); michael@0: } michael@0: michael@0: return; michael@0: } michael@0: michael@0: void michael@0: MediaEngineWebRTCAudioSource::GetUUID(nsAString& aUUID) michael@0: { michael@0: if (mInitDone) { michael@0: aUUID.Assign(mDeviceUUID); michael@0: } michael@0: michael@0: return; michael@0: } michael@0: michael@0: nsresult michael@0: MediaEngineWebRTCAudioSource::Config(bool aEchoOn, uint32_t aEcho, michael@0: bool aAgcOn, uint32_t aAGC, michael@0: bool aNoiseOn, uint32_t aNoise, michael@0: int32_t aPlayoutDelay) michael@0: { michael@0: LOG(("Audio config: aec: %d, agc: %d, noise: %d", michael@0: aEchoOn ? aEcho : -1, michael@0: aAgcOn ? aAGC : -1, michael@0: aNoiseOn ? aNoise : -1)); michael@0: michael@0: bool update_echo = (mEchoOn != aEchoOn); michael@0: bool update_agc = (mAgcOn != aAgcOn); michael@0: bool update_noise = (mNoiseOn != aNoiseOn); michael@0: mEchoOn = aEchoOn; michael@0: mAgcOn = aAgcOn; michael@0: mNoiseOn = aNoiseOn; michael@0: michael@0: if ((webrtc::EcModes) aEcho != webrtc::kEcUnchanged) { michael@0: if (mEchoCancel != (webrtc::EcModes) aEcho) { michael@0: update_echo = true; michael@0: mEchoCancel = (webrtc::EcModes) aEcho; michael@0: } michael@0: } michael@0: if ((webrtc::AgcModes) aAGC != webrtc::kAgcUnchanged) { michael@0: if (mAGC != (webrtc::AgcModes) aAGC) { michael@0: update_agc = true; michael@0: mAGC = (webrtc::AgcModes) aAGC; michael@0: } michael@0: } michael@0: if ((webrtc::NsModes) aNoise != webrtc::kNsUnchanged) { michael@0: if (mNoiseSuppress != (webrtc::NsModes) aNoise) { michael@0: update_noise = true; michael@0: mNoiseSuppress = (webrtc::NsModes) aNoise; michael@0: } michael@0: } michael@0: mPlayoutDelay = aPlayoutDelay; michael@0: michael@0: if (mInitDone) { michael@0: int error; michael@0: michael@0: if (update_echo && michael@0: 0 != (error = mVoEProcessing->SetEcStatus(mEchoOn, (webrtc::EcModes) aEcho))) { michael@0: LOG(("%s Error setting Echo Status: %d ",__FUNCTION__, error)); michael@0: // Overhead of capturing all the time is very low (<0.1% of an audio only call) michael@0: if (mEchoOn) { michael@0: if (0 != (error = mVoEProcessing->SetEcMetricsStatus(true))) { michael@0: LOG(("%s Error setting Echo Metrics: %d ",__FUNCTION__, error)); michael@0: } michael@0: } michael@0: } michael@0: if (update_agc && michael@0: 0 != (error = mVoEProcessing->SetAgcStatus(mAgcOn, (webrtc::AgcModes) aAGC))) { michael@0: LOG(("%s Error setting AGC Status: %d ",__FUNCTION__, error)); michael@0: } michael@0: if (update_noise && michael@0: 0 != (error = mVoEProcessing->SetNsStatus(mNoiseOn, (webrtc::NsModes) aNoise))) { michael@0: LOG(("%s Error setting NoiseSuppression Status: %d ",__FUNCTION__, error)); michael@0: } michael@0: } michael@0: return NS_OK; michael@0: } michael@0: michael@0: nsresult michael@0: MediaEngineWebRTCAudioSource::Allocate(const AudioTrackConstraintsN &aConstraints, michael@0: const MediaEnginePrefs &aPrefs) michael@0: { michael@0: if (mState == kReleased) { michael@0: if (mInitDone) { michael@0: ScopedCustomReleasePtr ptrVoEHw(webrtc::VoEHardware::GetInterface(mVoiceEngine)); michael@0: if (!ptrVoEHw || ptrVoEHw->SetRecordingDevice(mCapIndex)) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: mState = kAllocated; michael@0: LOG(("Audio device %d allocated", mCapIndex)); michael@0: } else { michael@0: LOG(("Audio device is not initalized")); michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: } else if (mSources.IsEmpty()) { michael@0: LOG(("Audio device %d reallocated", mCapIndex)); michael@0: } else { michael@0: LOG(("Audio device %d allocated shared", mCapIndex)); michael@0: } michael@0: return NS_OK; michael@0: } michael@0: michael@0: nsresult michael@0: MediaEngineWebRTCAudioSource::Deallocate() michael@0: { michael@0: if (mSources.IsEmpty()) { michael@0: if (mState != kStopped && mState != kAllocated) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: michael@0: mState = kReleased; michael@0: LOG(("Audio device %d deallocated", mCapIndex)); michael@0: } else { michael@0: LOG(("Audio device %d deallocated but still in use", mCapIndex)); michael@0: } michael@0: return NS_OK; michael@0: } michael@0: michael@0: nsresult michael@0: MediaEngineWebRTCAudioSource::Start(SourceMediaStream* aStream, TrackID aID) michael@0: { michael@0: if (!mInitDone || !aStream) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: michael@0: { michael@0: MonitorAutoLock lock(mMonitor); michael@0: mSources.AppendElement(aStream); michael@0: } michael@0: michael@0: AudioSegment* segment = new AudioSegment(); michael@0: aStream->AddTrack(aID, SAMPLE_FREQUENCY, 0, segment); michael@0: aStream->AdvanceKnownTracksTime(STREAM_TIME_MAX); michael@0: // XXX Make this based on the pref. michael@0: aStream->RegisterForAudioMixing(); michael@0: LOG(("Start audio for stream %p", aStream)); michael@0: michael@0: if (mState == kStarted) { michael@0: MOZ_ASSERT(aID == mTrackID); michael@0: return NS_OK; michael@0: } michael@0: mState = kStarted; michael@0: mTrackID = aID; michael@0: michael@0: // Make sure logger starts before capture michael@0: AsyncLatencyLogger::Get(true); michael@0: michael@0: // Register output observer michael@0: // XXX michael@0: MOZ_ASSERT(gFarendObserver); michael@0: gFarendObserver->Clear(); michael@0: michael@0: // Configure audio processing in webrtc code michael@0: Config(mEchoOn, webrtc::kEcUnchanged, michael@0: mAgcOn, webrtc::kAgcUnchanged, michael@0: mNoiseOn, webrtc::kNsUnchanged, michael@0: mPlayoutDelay); michael@0: michael@0: if (mVoEBase->StartReceive(mChannel)) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: if (mVoEBase->StartSend(mChannel)) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: michael@0: // Attach external media processor, so this::Process will be called. michael@0: mVoERender->RegisterExternalMediaProcessing(mChannel, webrtc::kRecordingPerChannel, *this); michael@0: michael@0: return NS_OK; michael@0: } michael@0: michael@0: nsresult michael@0: MediaEngineWebRTCAudioSource::Stop(SourceMediaStream *aSource, TrackID aID) michael@0: { michael@0: { michael@0: MonitorAutoLock lock(mMonitor); michael@0: michael@0: if (!mSources.RemoveElement(aSource)) { michael@0: // Already stopped - this is allowed michael@0: return NS_OK; michael@0: } michael@0: if (!mSources.IsEmpty()) { michael@0: return NS_OK; michael@0: } michael@0: if (mState != kStarted) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: if (!mVoEBase) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: michael@0: mState = kStopped; michael@0: aSource->EndTrack(aID); michael@0: } michael@0: michael@0: mVoERender->DeRegisterExternalMediaProcessing(mChannel, webrtc::kRecordingPerChannel); michael@0: michael@0: if (mVoEBase->StopSend(mChannel)) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: if (mVoEBase->StopReceive(mChannel)) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: return NS_OK; michael@0: } michael@0: michael@0: void michael@0: MediaEngineWebRTCAudioSource::NotifyPull(MediaStreamGraph* aGraph, michael@0: SourceMediaStream *aSource, michael@0: TrackID aID, michael@0: StreamTime aDesiredTime, michael@0: TrackTicks &aLastEndTime) michael@0: { michael@0: // Ignore - we push audio data michael@0: #ifdef DEBUG michael@0: TrackTicks target = TimeToTicksRoundUp(SAMPLE_FREQUENCY, aDesiredTime); michael@0: TrackTicks delta = target - aLastEndTime; michael@0: LOG(("Audio: NotifyPull: aDesiredTime %ld, target %ld, delta %ld",(int64_t) aDesiredTime, (int64_t) target, (int64_t) delta)); michael@0: aLastEndTime = target; michael@0: #endif michael@0: } michael@0: michael@0: nsresult michael@0: MediaEngineWebRTCAudioSource::Snapshot(uint32_t aDuration, nsIDOMFile** aFile) michael@0: { michael@0: return NS_ERROR_NOT_IMPLEMENTED; michael@0: } michael@0: michael@0: void michael@0: MediaEngineWebRTCAudioSource::Init() michael@0: { michael@0: mVoEBase = webrtc::VoEBase::GetInterface(mVoiceEngine); michael@0: michael@0: mVoEBase->Init(); michael@0: michael@0: mVoERender = webrtc::VoEExternalMedia::GetInterface(mVoiceEngine); michael@0: if (!mVoERender) { michael@0: return; michael@0: } michael@0: mVoENetwork = webrtc::VoENetwork::GetInterface(mVoiceEngine); michael@0: if (!mVoENetwork) { michael@0: return; michael@0: } michael@0: michael@0: mVoEProcessing = webrtc::VoEAudioProcessing::GetInterface(mVoiceEngine); michael@0: if (!mVoEProcessing) { michael@0: return; michael@0: } michael@0: michael@0: mVoECallReport = webrtc::VoECallReport::GetInterface(mVoiceEngine); michael@0: if (!mVoECallReport) { michael@0: return; michael@0: } michael@0: michael@0: mChannel = mVoEBase->CreateChannel(); michael@0: if (mChannel < 0) { michael@0: return; michael@0: } michael@0: mNullTransport = new NullTransport(); michael@0: if (mVoENetwork->RegisterExternalTransport(mChannel, *mNullTransport)) { michael@0: return; michael@0: } michael@0: michael@0: // Check for availability. michael@0: ScopedCustomReleasePtr ptrVoEHw(webrtc::VoEHardware::GetInterface(mVoiceEngine)); michael@0: if (!ptrVoEHw || ptrVoEHw->SetRecordingDevice(mCapIndex)) { michael@0: return; michael@0: } michael@0: michael@0: #ifndef MOZ_B2G michael@0: // Because of the permission mechanism of B2G, we need to skip the status michael@0: // check here. michael@0: bool avail = false; michael@0: ptrVoEHw->GetRecordingDeviceStatus(avail); michael@0: if (!avail) { michael@0: return; michael@0: } michael@0: #endif // MOZ_B2G michael@0: michael@0: // Set "codec" to PCM, 32kHz on 1 channel michael@0: ScopedCustomReleasePtr ptrVoECodec(webrtc::VoECodec::GetInterface(mVoiceEngine)); michael@0: if (!ptrVoECodec) { michael@0: return; michael@0: } michael@0: michael@0: webrtc::CodecInst codec; michael@0: strcpy(codec.plname, ENCODING); michael@0: codec.channels = CHANNELS; michael@0: codec.rate = SAMPLE_RATE; michael@0: codec.plfreq = SAMPLE_FREQUENCY; michael@0: codec.pacsize = SAMPLE_LENGTH; michael@0: codec.pltype = 0; // Default payload type michael@0: michael@0: if (!ptrVoECodec->SetSendCodec(mChannel, codec)) { michael@0: mInitDone = true; michael@0: } michael@0: } michael@0: michael@0: void michael@0: MediaEngineWebRTCAudioSource::Shutdown() michael@0: { michael@0: if (!mInitDone) { michael@0: // duplicate these here in case we failed during Init() michael@0: if (mChannel != -1) { michael@0: mVoENetwork->DeRegisterExternalTransport(mChannel); michael@0: } michael@0: michael@0: delete mNullTransport; michael@0: return; michael@0: } michael@0: michael@0: if (mState == kStarted) { michael@0: while (!mSources.IsEmpty()) { michael@0: Stop(mSources[0], kAudioTrack); // XXX change to support multiple tracks michael@0: } michael@0: MOZ_ASSERT(mState == kStopped); michael@0: } michael@0: michael@0: if (mState == kAllocated || mState == kStopped) { michael@0: Deallocate(); michael@0: } michael@0: michael@0: mVoEBase->Terminate(); michael@0: if (mChannel != -1) { michael@0: mVoENetwork->DeRegisterExternalTransport(mChannel); michael@0: } michael@0: michael@0: delete mNullTransport; michael@0: michael@0: mVoEProcessing = nullptr; michael@0: mVoENetwork = nullptr; michael@0: mVoERender = nullptr; michael@0: mVoEBase = nullptr; michael@0: michael@0: mState = kReleased; michael@0: mInitDone = false; michael@0: } michael@0: michael@0: typedef int16_t sample; michael@0: michael@0: void michael@0: MediaEngineWebRTCAudioSource::Process(int channel, michael@0: webrtc::ProcessingTypes type, sample* audio10ms, michael@0: int length, int samplingFreq, bool isStereo) michael@0: { michael@0: // On initial capture, throw away all far-end data except the most recent sample michael@0: // since it's already irrelevant and we want to keep avoid confusing the AEC far-end michael@0: // input code with "old" audio. michael@0: if (!mStarted) { michael@0: mStarted = true; michael@0: while (gFarendObserver->Size() > 1) { michael@0: FarEndAudioChunk *buffer = gFarendObserver->Pop(); // only call if size() > 0 michael@0: free(buffer); michael@0: } michael@0: } michael@0: michael@0: while (gFarendObserver->Size() > 0) { michael@0: FarEndAudioChunk *buffer = gFarendObserver->Pop(); // only call if size() > 0 michael@0: if (buffer) { michael@0: int length = buffer->mSamples; michael@0: if (mVoERender->ExternalPlayoutData(buffer->mData, michael@0: gFarendObserver->PlayoutFrequency(), michael@0: gFarendObserver->PlayoutChannels(), michael@0: mPlayoutDelay, michael@0: length) == -1) { michael@0: return; michael@0: } michael@0: } michael@0: free(buffer); michael@0: } michael@0: michael@0: #ifdef PR_LOGGING michael@0: mSamples += length; michael@0: if (mSamples > samplingFreq) { michael@0: mSamples %= samplingFreq; // just in case mSamples >> samplingFreq michael@0: if (PR_LOG_TEST(GetMediaManagerLog(), PR_LOG_DEBUG)) { michael@0: webrtc::EchoStatistics echo; michael@0: michael@0: mVoECallReport->GetEchoMetricSummary(echo); michael@0: #define DUMP_STATVAL(x) (x).min, (x).max, (x).average michael@0: LOG(("Echo: ERL: %d/%d/%d, ERLE: %d/%d/%d, RERL: %d/%d/%d, NLP: %d/%d/%d", michael@0: DUMP_STATVAL(echo.erl), michael@0: DUMP_STATVAL(echo.erle), michael@0: DUMP_STATVAL(echo.rerl), michael@0: DUMP_STATVAL(echo.a_nlp))); michael@0: } michael@0: } michael@0: #endif michael@0: michael@0: MonitorAutoLock lock(mMonitor); michael@0: if (mState != kStarted) michael@0: return; michael@0: michael@0: uint32_t len = mSources.Length(); michael@0: for (uint32_t i = 0; i < len; i++) { michael@0: nsRefPtr buffer = SharedBuffer::Create(length * sizeof(sample)); michael@0: michael@0: sample* dest = static_cast(buffer->Data()); michael@0: memcpy(dest, audio10ms, length * sizeof(sample)); michael@0: michael@0: AudioSegment segment; michael@0: nsAutoTArray channels; michael@0: channels.AppendElement(dest); michael@0: segment.AppendFrames(buffer.forget(), channels, length); michael@0: TimeStamp insertTime; michael@0: segment.GetStartTime(insertTime); michael@0: michael@0: SourceMediaStream *source = mSources[i]; michael@0: if (source) { michael@0: // This is safe from any thread, and is safe if the track is Finished michael@0: // or Destroyed. michael@0: // Make sure we include the stream and the track. michael@0: // The 0:1 is a flag to note when we've done the final insert for a given input block. michael@0: LogTime(AsyncLatencyLogger::AudioTrackInsertion, LATENCY_STREAM_ID(source, mTrackID), michael@0: (i+1 < len) ? 0 : 1, insertTime); michael@0: michael@0: source->AppendToTrack(mTrackID, &segment); michael@0: } michael@0: } michael@0: michael@0: return; michael@0: } michael@0: michael@0: }