michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef MOZILLA_AUDIOSEGMENT_H_
michael@0: #define MOZILLA_AUDIOSEGMENT_H_
michael@0: 
michael@0: #include "MediaSegment.h"
michael@0: #include "AudioSampleFormat.h"
michael@0: #include "SharedBuffer.h"
michael@0: #include "WebAudioUtils.h"
michael@0: #ifdef MOZILLA_INTERNAL_API
michael@0: #include "mozilla/TimeStamp.h"
michael@0: #endif
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: template<typename T>
michael@0: class SharedChannelArrayBuffer : public ThreadSharedObject {
michael@0: public:
michael@0:   SharedChannelArrayBuffer(nsTArray<nsTArray<T> >* aBuffers)
michael@0:   {
michael@0:     mBuffers.SwapElements(*aBuffers);
michael@0:   }
michael@0: 
michael@0:   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
michael@0:   {
michael@0:     size_t amount = 0;
michael@0:     amount += mBuffers.SizeOfExcludingThis(aMallocSizeOf);
michael@0:     for (size_t i = 0; i < mBuffers.Length(); i++) {
michael@0:       amount += mBuffers[i].SizeOfExcludingThis(aMallocSizeOf);
michael@0:     }
michael@0: 
michael@0:     return amount;
michael@0:   }
michael@0: 
michael@0:   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
michael@0:   {
michael@0:     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
michael@0:   }
michael@0: 
michael@0:   nsTArray<nsTArray<T> > mBuffers;
michael@0: };
michael@0: 
michael@0: class AudioStream;
michael@0: class AudioMixer;
michael@0: 
michael@0: /**
michael@0:  * For auto-arrays etc, guess this as the common number of channels.
michael@0:  */
michael@0: const int GUESS_AUDIO_CHANNELS = 2;
michael@0: 
michael@0: // We ensure that the graph advances in steps that are multiples of the Web
michael@0: // Audio block size
michael@0: const uint32_t WEBAUDIO_BLOCK_SIZE_BITS = 7;
michael@0: const uint32_t WEBAUDIO_BLOCK_SIZE = 1 << WEBAUDIO_BLOCK_SIZE_BITS;
michael@0: 
michael@0: void InterleaveAndConvertBuffer(const void** aSourceChannels,
michael@0:                                 AudioSampleFormat aSourceFormat,
michael@0:                                 int32_t aLength, float aVolume,
michael@0:                                 int32_t aChannels,
michael@0:                                 AudioDataValue* aOutput);
michael@0: 
michael@0: /**
michael@0:  * Given an array of input channels (aChannelData), downmix to aOutputChannels,
michael@0:  * interleave the channel data. A total of aOutputChannels*aDuration
michael@0:  * interleaved samples will be copied to a channel buffer in aOutput.
michael@0:  */
michael@0: void DownmixAndInterleave(const nsTArray<const void*>& aChannelData,
michael@0:                           AudioSampleFormat aSourceFormat, int32_t aDuration,
michael@0:                           float aVolume, uint32_t aOutputChannels,
michael@0:                           AudioDataValue* aOutput);
michael@0: 
michael@0: /**
michael@0:  * An AudioChunk represents a multi-channel buffer of audio samples.
michael@0:  * It references an underlying ThreadSharedObject which manages the lifetime
michael@0:  * of the buffer. An AudioChunk maintains its own duration and channel data
michael@0:  * pointers so it can represent a subinterval of a buffer without copying.
michael@0:  * An AudioChunk can store its individual channels anywhere; it maintains
michael@0:  * separate pointers to each channel's buffer.
michael@0:  */
michael@0: struct AudioChunk {
michael@0:   typedef mozilla::AudioSampleFormat SampleFormat;
michael@0: 
michael@0:   // Generic methods
michael@0:   void SliceTo(TrackTicks aStart, TrackTicks aEnd)
michael@0:   {
michael@0:     NS_ASSERTION(aStart >= 0 && aStart < aEnd && aEnd <= mDuration,
michael@0:                  "Slice out of bounds");
michael@0:     if (mBuffer) {
michael@0:       MOZ_ASSERT(aStart < INT32_MAX, "Can't slice beyond 32-bit sample lengths");
michael@0:       for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) {
michael@0:         mChannelData[channel] = AddAudioSampleOffset(mChannelData[channel],
michael@0:             mBufferFormat, int32_t(aStart));
michael@0:       }
michael@0:     }
michael@0:     mDuration = aEnd - aStart;
michael@0:   }
michael@0:   TrackTicks GetDuration() const { return mDuration; }
michael@0:   bool CanCombineWithFollowing(const AudioChunk& aOther) const
michael@0:   {
michael@0:     if (aOther.mBuffer != mBuffer) {
michael@0:       return false;
michael@0:     }
michael@0:     if (mBuffer) {
michael@0:       NS_ASSERTION(aOther.mBufferFormat == mBufferFormat,
michael@0:                    "Wrong metadata about buffer");
michael@0:       NS_ASSERTION(aOther.mChannelData.Length() == mChannelData.Length(),
michael@0:                    "Mismatched channel count");
michael@0:       if (mDuration > INT32_MAX) {
michael@0:         return false;
michael@0:       }
michael@0:       for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) {
michael@0:         if (aOther.mChannelData[channel] != AddAudioSampleOffset(mChannelData[channel],
michael@0:             mBufferFormat, int32_t(mDuration))) {
michael@0:           return false;
michael@0:         }
michael@0:       }
michael@0:     }
michael@0:     return true;
michael@0:   }
michael@0:   bool IsNull() const { return mBuffer == nullptr; }
michael@0:   void SetNull(TrackTicks aDuration)
michael@0:   {
michael@0:     mBuffer = nullptr;
michael@0:     mChannelData.Clear();
michael@0:     mDuration = aDuration;
michael@0:     mVolume = 1.0f;
michael@0:     mBufferFormat = AUDIO_FORMAT_SILENCE;
michael@0:   }
michael@0:   int ChannelCount() const { return mChannelData.Length(); }
michael@0: 
michael@0:   size_t SizeOfExcludingThisIfUnshared(MallocSizeOf aMallocSizeOf) const
michael@0:   {
michael@0:     return SizeOfExcludingThis(aMallocSizeOf, true);
michael@0:   }
michael@0: 
michael@0:   size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf, bool aUnshared) const
michael@0:   {
michael@0:     size_t amount = 0;
michael@0: 
michael@0:     // Possibly owned:
michael@0:     // - mBuffer - Can hold data that is also in the decoded audio queue. If it
michael@0:     //             is not shared, or unshared == false it gets counted.
michael@0:     if (mBuffer && (!aUnshared || !mBuffer->IsShared())) {
michael@0:       amount += mBuffer->SizeOfIncludingThis(aMallocSizeOf);
michael@0:     }
michael@0: 
michael@0:     // Memory in the array is owned by mBuffer.
michael@0:     amount += mChannelData.SizeOfExcludingThis(aMallocSizeOf);
michael@0:     return amount;
michael@0:   }
michael@0: 
michael@0:   TrackTicks mDuration; // in frames within the buffer
michael@0:   nsRefPtr<ThreadSharedObject> mBuffer; // the buffer object whose lifetime is managed; null means data is all zeroes
michael@0:   nsTArray<const void*> mChannelData; // one pointer per channel; empty if and only if mBuffer is null
michael@0:   float mVolume; // volume multiplier to apply (1.0f if mBuffer is nonnull)
michael@0:   SampleFormat mBufferFormat; // format of frames in mBuffer (only meaningful if mBuffer is nonnull)
michael@0: #ifdef MOZILLA_INTERNAL_API
michael@0:   mozilla::TimeStamp mTimeStamp;           // time at which this has been fetched from the MediaEngine
michael@0: #endif
michael@0: };
michael@0: 
michael@0: 
michael@0: /**
michael@0:  * A list of audio samples consisting of a sequence of slices of SharedBuffers.
michael@0:  * The audio rate is determined by the track, not stored in this class.
michael@0:  */
michael@0: class AudioSegment : public MediaSegmentBase<AudioSegment, AudioChunk> {
michael@0: public:
michael@0:   typedef mozilla::AudioSampleFormat SampleFormat;
michael@0: 
michael@0:   AudioSegment() : MediaSegmentBase<AudioSegment, AudioChunk>(AUDIO) {}
michael@0: 
michael@0:   // Resample the whole segment in place.
michael@0:   template<typename T>
michael@0:   void Resample(SpeexResamplerState* aResampler, uint32_t aInRate, uint32_t aOutRate)
michael@0:   {
michael@0:     mDuration = 0;
michael@0: 
michael@0:     for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
michael@0:       nsAutoTArray<nsTArray<T>, GUESS_AUDIO_CHANNELS> output;
michael@0:       nsAutoTArray<const T*, GUESS_AUDIO_CHANNELS> bufferPtrs;
michael@0:       AudioChunk& c = *ci;
michael@0:       // If this chunk is null, don't bother resampling, just alter its duration
michael@0:       if (c.IsNull()) {
michael@0:         c.mDuration = (c.mDuration * aOutRate) / aInRate;
michael@0:         mDuration += c.mDuration;
michael@0:         continue;
michael@0:       }
michael@0:       uint32_t channels = c.mChannelData.Length();
michael@0:       output.SetLength(channels);
michael@0:       bufferPtrs.SetLength(channels);
michael@0:       uint32_t inFrames = c.mDuration;
michael@0:       // Round up to allocate; the last frame may not be used.
michael@0:       NS_ASSERTION((UINT32_MAX - aInRate + 1) / c.mDuration >= aOutRate,
michael@0:                    "Dropping samples");
michael@0:       uint32_t outSize = (c.mDuration * aOutRate + aInRate - 1) / aInRate;
michael@0:       for (uint32_t i = 0; i < channels; i++) {
michael@0:         const T* in = static_cast<const T*>(c.mChannelData[i]);
michael@0:         T* out = output[i].AppendElements(outSize);
michael@0:         uint32_t outFrames = outSize;
michael@0: 
michael@0:         dom::WebAudioUtils::SpeexResamplerProcess(aResampler, i,
michael@0:                                                   in, &inFrames,
michael@0:                                                   out, &outFrames);
michael@0:         MOZ_ASSERT(inFrames == c.mDuration);
michael@0:         bufferPtrs[i] = out;
michael@0:         output[i].SetLength(outFrames);
michael@0:       }
michael@0:       MOZ_ASSERT(channels > 0);
michael@0:       c.mDuration = output[0].Length();
michael@0:       c.mBuffer = new mozilla::SharedChannelArrayBuffer<T>(&output);
michael@0:       for (uint32_t i = 0; i < channels; i++) {
michael@0:         c.mChannelData[i] = bufferPtrs[i];
michael@0:       }
michael@0:       mDuration += c.mDuration;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   void ResampleChunks(SpeexResamplerState* aResampler);
michael@0: 
michael@0:   void AppendFrames(already_AddRefed<ThreadSharedObject> aBuffer,
michael@0:                     const nsTArray<const float*>& aChannelData,
michael@0:                     int32_t aDuration)
michael@0:   {
michael@0:     AudioChunk* chunk = AppendChunk(aDuration);
michael@0:     chunk->mBuffer = aBuffer;
michael@0:     for (uint32_t channel = 0; channel < aChannelData.Length(); ++channel) {
michael@0:       chunk->mChannelData.AppendElement(aChannelData[channel]);
michael@0:     }
michael@0:     chunk->mVolume = 1.0f;
michael@0:     chunk->mBufferFormat = AUDIO_FORMAT_FLOAT32;
michael@0: #ifdef MOZILLA_INTERNAL_API
michael@0:     chunk->mTimeStamp = TimeStamp::Now();
michael@0: #endif
michael@0:   }
michael@0:   void AppendFrames(already_AddRefed<ThreadSharedObject> aBuffer,
michael@0:                     const nsTArray<const int16_t*>& aChannelData,
michael@0:                     int32_t aDuration)
michael@0:   {
michael@0:     AudioChunk* chunk = AppendChunk(aDuration);
michael@0:     chunk->mBuffer = aBuffer;
michael@0:     for (uint32_t channel = 0; channel < aChannelData.Length(); ++channel) {
michael@0:       chunk->mChannelData.AppendElement(aChannelData[channel]);
michael@0:     }
michael@0:     chunk->mVolume = 1.0f;
michael@0:     chunk->mBufferFormat = AUDIO_FORMAT_S16;
michael@0: #ifdef MOZILLA_INTERNAL_API
michael@0:     chunk->mTimeStamp = TimeStamp::Now();
michael@0: #endif
michael@0:   }
michael@0:   // Consumes aChunk, and returns a pointer to the persistent copy of aChunk
michael@0:   // in the segment.
michael@0:   AudioChunk* AppendAndConsumeChunk(AudioChunk* aChunk)
michael@0:   {
michael@0:     AudioChunk* chunk = AppendChunk(aChunk->mDuration);
michael@0:     chunk->mBuffer = aChunk->mBuffer.forget();
michael@0:     chunk->mChannelData.SwapElements(aChunk->mChannelData);
michael@0:     chunk->mVolume = aChunk->mVolume;
michael@0:     chunk->mBufferFormat = aChunk->mBufferFormat;
michael@0: #ifdef MOZILLA_INTERNAL_API
michael@0:     chunk->mTimeStamp = TimeStamp::Now();
michael@0: #endif
michael@0:     return chunk;
michael@0:   }
michael@0:   void ApplyVolume(float aVolume);
michael@0:   void WriteTo(uint64_t aID, AudioStream* aOutput, AudioMixer* aMixer = nullptr);
michael@0: 
michael@0:   int ChannelCount() {
michael@0:     NS_WARN_IF_FALSE(!mChunks.IsEmpty(),
michael@0:         "Cannot query channel count on a AudioSegment with no chunks.");
michael@0:     // Find the first chunk that has non-zero channels. A chunk that hs zero
michael@0:     // channels is just silence and we can simply discard it.
michael@0:     for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
michael@0:       if (ci->ChannelCount()) {
michael@0:         return ci->ChannelCount();
michael@0:       }
michael@0:     }
michael@0:     return 0;
michael@0:   }
michael@0: 
michael@0:   static Type StaticType() { return AUDIO; }
michael@0: 
michael@0:   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
michael@0:   {
michael@0:     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
michael@0:   }
michael@0: };
michael@0: 
michael@0: }
michael@0: 
michael@0: #endif /* MOZILLA_AUDIOSEGMENT_H_ */