1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/content/media/AudioSegment.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,294 @@
1.4 +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
1.5 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.6 + * License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.7 + * You can obtain one at http://mozilla.org/MPL/2.0/. */
1.8 +
1.9 +#ifndef MOZILLA_AUDIOSEGMENT_H_
1.10 +#define MOZILLA_AUDIOSEGMENT_H_
1.11 +
1.12 +#include "MediaSegment.h"
1.13 +#include "AudioSampleFormat.h"
1.14 +#include "SharedBuffer.h"
1.15 +#include "WebAudioUtils.h"
1.16 +#ifdef MOZILLA_INTERNAL_API
1.17 +#include "mozilla/TimeStamp.h"
1.18 +#endif
1.19 +
1.20 +namespace mozilla {
1.21 +
1.22 +template<typename T>
1.23 +class SharedChannelArrayBuffer : public ThreadSharedObject {
1.24 +public:
1.25 + SharedChannelArrayBuffer(nsTArray<nsTArray<T> >* aBuffers)
1.26 + {
1.27 + mBuffers.SwapElements(*aBuffers);
1.28 + }
1.29 +
1.30 + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
1.31 + {
1.32 + size_t amount = 0;
1.33 + amount += mBuffers.SizeOfExcludingThis(aMallocSizeOf);
1.34 + for (size_t i = 0; i < mBuffers.Length(); i++) {
1.35 + amount += mBuffers[i].SizeOfExcludingThis(aMallocSizeOf);
1.36 + }
1.37 +
1.38 + return amount;
1.39 + }
1.40 +
1.41 + virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
1.42 + {
1.43 + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
1.44 + }
1.45 +
1.46 + nsTArray<nsTArray<T> > mBuffers;
1.47 +};
1.48 +
1.49 +class AudioStream;
1.50 +class AudioMixer;
1.51 +
1.52 +/**
1.53 + * For auto-arrays etc, guess this as the common number of channels.
1.54 + */
1.55 +const int GUESS_AUDIO_CHANNELS = 2;
1.56 +
1.57 +// We ensure that the graph advances in steps that are multiples of the Web
1.58 +// Audio block size
1.59 +const uint32_t WEBAUDIO_BLOCK_SIZE_BITS = 7;
1.60 +const uint32_t WEBAUDIO_BLOCK_SIZE = 1 << WEBAUDIO_BLOCK_SIZE_BITS;
1.61 +
1.62 +void InterleaveAndConvertBuffer(const void** aSourceChannels,
1.63 + AudioSampleFormat aSourceFormat,
1.64 + int32_t aLength, float aVolume,
1.65 + int32_t aChannels,
1.66 + AudioDataValue* aOutput);
1.67 +
1.68 +/**
1.69 + * Given an array of input channels (aChannelData), downmix to aOutputChannels,
1.70 + * interleave the channel data. A total of aOutputChannels*aDuration
1.71 + * interleaved samples will be copied to a channel buffer in aOutput.
1.72 + */
1.73 +void DownmixAndInterleave(const nsTArray<const void*>& aChannelData,
1.74 + AudioSampleFormat aSourceFormat, int32_t aDuration,
1.75 + float aVolume, uint32_t aOutputChannels,
1.76 + AudioDataValue* aOutput);
1.77 +
1.78 +/**
1.79 + * An AudioChunk represents a multi-channel buffer of audio samples.
1.80 + * It references an underlying ThreadSharedObject which manages the lifetime
1.81 + * of the buffer. An AudioChunk maintains its own duration and channel data
1.82 + * pointers so it can represent a subinterval of a buffer without copying.
1.83 + * An AudioChunk can store its individual channels anywhere; it maintains
1.84 + * separate pointers to each channel's buffer.
1.85 + */
1.86 +struct AudioChunk {
1.87 + typedef mozilla::AudioSampleFormat SampleFormat;
1.88 +
1.89 + // Generic methods
1.90 + void SliceTo(TrackTicks aStart, TrackTicks aEnd)
1.91 + {
1.92 + NS_ASSERTION(aStart >= 0 && aStart < aEnd && aEnd <= mDuration,
1.93 + "Slice out of bounds");
1.94 + if (mBuffer) {
1.95 + MOZ_ASSERT(aStart < INT32_MAX, "Can't slice beyond 32-bit sample lengths");
1.96 + for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) {
1.97 + mChannelData[channel] = AddAudioSampleOffset(mChannelData[channel],
1.98 + mBufferFormat, int32_t(aStart));
1.99 + }
1.100 + }
1.101 + mDuration = aEnd - aStart;
1.102 + }
1.103 + TrackTicks GetDuration() const { return mDuration; }
1.104 + bool CanCombineWithFollowing(const AudioChunk& aOther) const
1.105 + {
1.106 + if (aOther.mBuffer != mBuffer) {
1.107 + return false;
1.108 + }
1.109 + if (mBuffer) {
1.110 + NS_ASSERTION(aOther.mBufferFormat == mBufferFormat,
1.111 + "Wrong metadata about buffer");
1.112 + NS_ASSERTION(aOther.mChannelData.Length() == mChannelData.Length(),
1.113 + "Mismatched channel count");
1.114 + if (mDuration > INT32_MAX) {
1.115 + return false;
1.116 + }
1.117 + for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) {
1.118 + if (aOther.mChannelData[channel] != AddAudioSampleOffset(mChannelData[channel],
1.119 + mBufferFormat, int32_t(mDuration))) {
1.120 + return false;
1.121 + }
1.122 + }
1.123 + }
1.124 + return true;
1.125 + }
1.126 + bool IsNull() const { return mBuffer == nullptr; }
1.127 + void SetNull(TrackTicks aDuration)
1.128 + {
1.129 + mBuffer = nullptr;
1.130 + mChannelData.Clear();
1.131 + mDuration = aDuration;
1.132 + mVolume = 1.0f;
1.133 + mBufferFormat = AUDIO_FORMAT_SILENCE;
1.134 + }
1.135 + int ChannelCount() const { return mChannelData.Length(); }
1.136 +
1.137 + size_t SizeOfExcludingThisIfUnshared(MallocSizeOf aMallocSizeOf) const
1.138 + {
1.139 + return SizeOfExcludingThis(aMallocSizeOf, true);
1.140 + }
1.141 +
1.142 + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf, bool aUnshared) const
1.143 + {
1.144 + size_t amount = 0;
1.145 +
1.146 + // Possibly owned:
1.147 + // - mBuffer - Can hold data that is also in the decoded audio queue. If it
1.148 + // is not shared, or unshared == false it gets counted.
1.149 + if (mBuffer && (!aUnshared || !mBuffer->IsShared())) {
1.150 + amount += mBuffer->SizeOfIncludingThis(aMallocSizeOf);
1.151 + }
1.152 +
1.153 + // Memory in the array is owned by mBuffer.
1.154 + amount += mChannelData.SizeOfExcludingThis(aMallocSizeOf);
1.155 + return amount;
1.156 + }
1.157 +
1.158 + TrackTicks mDuration; // in frames within the buffer
1.159 + nsRefPtr<ThreadSharedObject> mBuffer; // the buffer object whose lifetime is managed; null means data is all zeroes
1.160 + nsTArray<const void*> mChannelData; // one pointer per channel; empty if and only if mBuffer is null
1.161 + float mVolume; // volume multiplier to apply (1.0f if mBuffer is nonnull)
1.162 + SampleFormat mBufferFormat; // format of frames in mBuffer (only meaningful if mBuffer is nonnull)
1.163 +#ifdef MOZILLA_INTERNAL_API
1.164 + mozilla::TimeStamp mTimeStamp; // time at which this has been fetched from the MediaEngine
1.165 +#endif
1.166 +};
1.167 +
1.168 +
1.169 +/**
1.170 + * A list of audio samples consisting of a sequence of slices of SharedBuffers.
1.171 + * The audio rate is determined by the track, not stored in this class.
1.172 + */
1.173 +class AudioSegment : public MediaSegmentBase<AudioSegment, AudioChunk> {
1.174 +public:
1.175 + typedef mozilla::AudioSampleFormat SampleFormat;
1.176 +
1.177 + AudioSegment() : MediaSegmentBase<AudioSegment, AudioChunk>(AUDIO) {}
1.178 +
1.179 + // Resample the whole segment in place.
1.180 + template<typename T>
1.181 + void Resample(SpeexResamplerState* aResampler, uint32_t aInRate, uint32_t aOutRate)
1.182 + {
1.183 + mDuration = 0;
1.184 +
1.185 + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
1.186 + nsAutoTArray<nsTArray<T>, GUESS_AUDIO_CHANNELS> output;
1.187 + nsAutoTArray<const T*, GUESS_AUDIO_CHANNELS> bufferPtrs;
1.188 + AudioChunk& c = *ci;
1.189 + // If this chunk is null, don't bother resampling, just alter its duration
1.190 + if (c.IsNull()) {
1.191 + c.mDuration = (c.mDuration * aOutRate) / aInRate;
1.192 + mDuration += c.mDuration;
1.193 + continue;
1.194 + }
1.195 + uint32_t channels = c.mChannelData.Length();
1.196 + output.SetLength(channels);
1.197 + bufferPtrs.SetLength(channels);
1.198 + uint32_t inFrames = c.mDuration;
1.199 + // Round up to allocate; the last frame may not be used.
1.200 + NS_ASSERTION((UINT32_MAX - aInRate + 1) / c.mDuration >= aOutRate,
1.201 + "Dropping samples");
1.202 + uint32_t outSize = (c.mDuration * aOutRate + aInRate - 1) / aInRate;
1.203 + for (uint32_t i = 0; i < channels; i++) {
1.204 + const T* in = static_cast<const T*>(c.mChannelData[i]);
1.205 + T* out = output[i].AppendElements(outSize);
1.206 + uint32_t outFrames = outSize;
1.207 +
1.208 + dom::WebAudioUtils::SpeexResamplerProcess(aResampler, i,
1.209 + in, &inFrames,
1.210 + out, &outFrames);
1.211 + MOZ_ASSERT(inFrames == c.mDuration);
1.212 + bufferPtrs[i] = out;
1.213 + output[i].SetLength(outFrames);
1.214 + }
1.215 + MOZ_ASSERT(channels > 0);
1.216 + c.mDuration = output[0].Length();
1.217 + c.mBuffer = new mozilla::SharedChannelArrayBuffer<T>(&output);
1.218 + for (uint32_t i = 0; i < channels; i++) {
1.219 + c.mChannelData[i] = bufferPtrs[i];
1.220 + }
1.221 + mDuration += c.mDuration;
1.222 + }
1.223 + }
1.224 +
1.225 + void ResampleChunks(SpeexResamplerState* aResampler);
1.226 +
1.227 + void AppendFrames(already_AddRefed<ThreadSharedObject> aBuffer,
1.228 + const nsTArray<const float*>& aChannelData,
1.229 + int32_t aDuration)
1.230 + {
1.231 + AudioChunk* chunk = AppendChunk(aDuration);
1.232 + chunk->mBuffer = aBuffer;
1.233 + for (uint32_t channel = 0; channel < aChannelData.Length(); ++channel) {
1.234 + chunk->mChannelData.AppendElement(aChannelData[channel]);
1.235 + }
1.236 + chunk->mVolume = 1.0f;
1.237 + chunk->mBufferFormat = AUDIO_FORMAT_FLOAT32;
1.238 +#ifdef MOZILLA_INTERNAL_API
1.239 + chunk->mTimeStamp = TimeStamp::Now();
1.240 +#endif
1.241 + }
1.242 + void AppendFrames(already_AddRefed<ThreadSharedObject> aBuffer,
1.243 + const nsTArray<const int16_t*>& aChannelData,
1.244 + int32_t aDuration)
1.245 + {
1.246 + AudioChunk* chunk = AppendChunk(aDuration);
1.247 + chunk->mBuffer = aBuffer;
1.248 + for (uint32_t channel = 0; channel < aChannelData.Length(); ++channel) {
1.249 + chunk->mChannelData.AppendElement(aChannelData[channel]);
1.250 + }
1.251 + chunk->mVolume = 1.0f;
1.252 + chunk->mBufferFormat = AUDIO_FORMAT_S16;
1.253 +#ifdef MOZILLA_INTERNAL_API
1.254 + chunk->mTimeStamp = TimeStamp::Now();
1.255 +#endif
1.256 + }
1.257 + // Consumes aChunk, and returns a pointer to the persistent copy of aChunk
1.258 + // in the segment.
1.259 + AudioChunk* AppendAndConsumeChunk(AudioChunk* aChunk)
1.260 + {
1.261 + AudioChunk* chunk = AppendChunk(aChunk->mDuration);
1.262 + chunk->mBuffer = aChunk->mBuffer.forget();
1.263 + chunk->mChannelData.SwapElements(aChunk->mChannelData);
1.264 + chunk->mVolume = aChunk->mVolume;
1.265 + chunk->mBufferFormat = aChunk->mBufferFormat;
1.266 +#ifdef MOZILLA_INTERNAL_API
1.267 + chunk->mTimeStamp = TimeStamp::Now();
1.268 +#endif
1.269 + return chunk;
1.270 + }
1.271 + void ApplyVolume(float aVolume);
1.272 + void WriteTo(uint64_t aID, AudioStream* aOutput, AudioMixer* aMixer = nullptr);
1.273 +
1.274 + int ChannelCount() {
1.275 + NS_WARN_IF_FALSE(!mChunks.IsEmpty(),
1.276 + "Cannot query channel count on a AudioSegment with no chunks.");
1.277 + // Find the first chunk that has non-zero channels. A chunk that hs zero
1.278 + // channels is just silence and we can simply discard it.
1.279 + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
1.280 + if (ci->ChannelCount()) {
1.281 + return ci->ChannelCount();
1.282 + }
1.283 + }
1.284 + return 0;
1.285 + }
1.286 +
1.287 + static Type StaticType() { return AUDIO; }
1.288 +
1.289 + virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
1.290 + {
1.291 + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
1.292 + }
1.293 +};
1.294 +
1.295 +}
1.296 +
1.297 +#endif /* MOZILLA_AUDIOSEGMENT_H_ */
