content/media/AudioSegment.h@97036ab72558
content/media/AudioSegment.h
Tue, 06 Jan 2015 21:39:09 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Tue, 06 Jan 2015 21:39:09 +0100
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
- permissions
- -rw-r--r--
Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MOZILLA_AUDIOSEGMENT_H_
7 #define MOZILLA_AUDIOSEGMENT_H_
9 #include "MediaSegment.h"
10 #include "AudioSampleFormat.h"
11 #include "SharedBuffer.h"
12 #include "WebAudioUtils.h"
13 #ifdef MOZILLA_INTERNAL_API
14 #include "mozilla/TimeStamp.h"
15 #endif
17 namespace mozilla {
19 template<typename T>
20 class SharedChannelArrayBuffer : public ThreadSharedObject {
21 public:
22 SharedChannelArrayBuffer(nsTArray<nsTArray<T> >* aBuffers)
23 {
24 mBuffers.SwapElements(*aBuffers);
25 }
27 virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
28 {
29 size_t amount = 0;
30 amount += mBuffers.SizeOfExcludingThis(aMallocSizeOf);
31 for (size_t i = 0; i < mBuffers.Length(); i++) {
32 amount += mBuffers[i].SizeOfExcludingThis(aMallocSizeOf);
33 }
35 return amount;
36 }
38 virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
39 {
40 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
41 }
43 nsTArray<nsTArray<T> > mBuffers;
44 };
46 class AudioStream;
47 class AudioMixer;
49 /**
50 * For auto-arrays etc, guess this as the common number of channels.
51 */
52 const int GUESS_AUDIO_CHANNELS = 2;
54 // We ensure that the graph advances in steps that are multiples of the Web
55 // Audio block size
56 const uint32_t WEBAUDIO_BLOCK_SIZE_BITS = 7;
57 const uint32_t WEBAUDIO_BLOCK_SIZE = 1 << WEBAUDIO_BLOCK_SIZE_BITS;
59 void InterleaveAndConvertBuffer(const void** aSourceChannels,
60 AudioSampleFormat aSourceFormat,
61 int32_t aLength, float aVolume,
62 int32_t aChannels,
63 AudioDataValue* aOutput);
65 /**
66 * Given an array of input channels (aChannelData), downmix to aOutputChannels,
67 * interleave the channel data. A total of aOutputChannels*aDuration
68 * interleaved samples will be copied to a channel buffer in aOutput.
69 */
70 void DownmixAndInterleave(const nsTArray<const void*>& aChannelData,
71 AudioSampleFormat aSourceFormat, int32_t aDuration,
72 float aVolume, uint32_t aOutputChannels,
73 AudioDataValue* aOutput);
75 /**
76 * An AudioChunk represents a multi-channel buffer of audio samples.
77 * It references an underlying ThreadSharedObject which manages the lifetime
78 * of the buffer. An AudioChunk maintains its own duration and channel data
79 * pointers so it can represent a subinterval of a buffer without copying.
80 * An AudioChunk can store its individual channels anywhere; it maintains
81 * separate pointers to each channel's buffer.
82 */
83 struct AudioChunk {
84 typedef mozilla::AudioSampleFormat SampleFormat;
86 // Generic methods
87 void SliceTo(TrackTicks aStart, TrackTicks aEnd)
88 {
89 NS_ASSERTION(aStart >= 0 && aStart < aEnd && aEnd <= mDuration,
90 "Slice out of bounds");
91 if (mBuffer) {
92 MOZ_ASSERT(aStart < INT32_MAX, "Can't slice beyond 32-bit sample lengths");
93 for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) {
94 mChannelData[channel] = AddAudioSampleOffset(mChannelData[channel],
95 mBufferFormat, int32_t(aStart));
96 }
97 }
98 mDuration = aEnd - aStart;
99 }
100 TrackTicks GetDuration() const { return mDuration; }
101 bool CanCombineWithFollowing(const AudioChunk& aOther) const
102 {
103 if (aOther.mBuffer != mBuffer) {
104 return false;
105 }
106 if (mBuffer) {
107 NS_ASSERTION(aOther.mBufferFormat == mBufferFormat,
108 "Wrong metadata about buffer");
109 NS_ASSERTION(aOther.mChannelData.Length() == mChannelData.Length(),
110 "Mismatched channel count");
111 if (mDuration > INT32_MAX) {
112 return false;
113 }
114 for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) {
115 if (aOther.mChannelData[channel] != AddAudioSampleOffset(mChannelData[channel],
116 mBufferFormat, int32_t(mDuration))) {
117 return false;
118 }
119 }
120 }
121 return true;
122 }
123 bool IsNull() const { return mBuffer == nullptr; }
124 void SetNull(TrackTicks aDuration)
125 {
126 mBuffer = nullptr;
127 mChannelData.Clear();
128 mDuration = aDuration;
129 mVolume = 1.0f;
130 mBufferFormat = AUDIO_FORMAT_SILENCE;
131 }
132 int ChannelCount() const { return mChannelData.Length(); }
134 size_t SizeOfExcludingThisIfUnshared(MallocSizeOf aMallocSizeOf) const
135 {
136 return SizeOfExcludingThis(aMallocSizeOf, true);
137 }
139 size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf, bool aUnshared) const
140 {
141 size_t amount = 0;
143 // Possibly owned:
144 // - mBuffer - Can hold data that is also in the decoded audio queue. If it
145 // is not shared, or unshared == false it gets counted.
146 if (mBuffer && (!aUnshared || !mBuffer->IsShared())) {
147 amount += mBuffer->SizeOfIncludingThis(aMallocSizeOf);
148 }
150 // Memory in the array is owned by mBuffer.
151 amount += mChannelData.SizeOfExcludingThis(aMallocSizeOf);
152 return amount;
153 }
155 TrackTicks mDuration; // in frames within the buffer
156 nsRefPtr<ThreadSharedObject> mBuffer; // the buffer object whose lifetime is managed; null means data is all zeroes
157 nsTArray<const void*> mChannelData; // one pointer per channel; empty if and only if mBuffer is null
158 float mVolume; // volume multiplier to apply (1.0f if mBuffer is nonnull)
159 SampleFormat mBufferFormat; // format of frames in mBuffer (only meaningful if mBuffer is nonnull)
160 #ifdef MOZILLA_INTERNAL_API
161 mozilla::TimeStamp mTimeStamp; // time at which this has been fetched from the MediaEngine
162 #endif
163 };
166 /**
167 * A list of audio samples consisting of a sequence of slices of SharedBuffers.
168 * The audio rate is determined by the track, not stored in this class.
169 */
170 class AudioSegment : public MediaSegmentBase<AudioSegment, AudioChunk> {
171 public:
172 typedef mozilla::AudioSampleFormat SampleFormat;
174 AudioSegment() : MediaSegmentBase<AudioSegment, AudioChunk>(AUDIO) {}
176 // Resample the whole segment in place.
177 template<typename T>
178 void Resample(SpeexResamplerState* aResampler, uint32_t aInRate, uint32_t aOutRate)
179 {
180 mDuration = 0;
182 for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
183 nsAutoTArray<nsTArray<T>, GUESS_AUDIO_CHANNELS> output;
184 nsAutoTArray<const T*, GUESS_AUDIO_CHANNELS> bufferPtrs;
185 AudioChunk& c = *ci;
186 // If this chunk is null, don't bother resampling, just alter its duration
187 if (c.IsNull()) {
188 c.mDuration = (c.mDuration * aOutRate) / aInRate;
189 mDuration += c.mDuration;
190 continue;
191 }
192 uint32_t channels = c.mChannelData.Length();
193 output.SetLength(channels);
194 bufferPtrs.SetLength(channels);
195 uint32_t inFrames = c.mDuration;
196 // Round up to allocate; the last frame may not be used.
197 NS_ASSERTION((UINT32_MAX - aInRate + 1) / c.mDuration >= aOutRate,
198 "Dropping samples");
199 uint32_t outSize = (c.mDuration * aOutRate + aInRate - 1) / aInRate;
200 for (uint32_t i = 0; i < channels; i++) {
201 const T* in = static_cast<const T*>(c.mChannelData[i]);
202 T* out = output[i].AppendElements(outSize);
203 uint32_t outFrames = outSize;
205 dom::WebAudioUtils::SpeexResamplerProcess(aResampler, i,
206 in, &inFrames,
207 out, &outFrames);
208 MOZ_ASSERT(inFrames == c.mDuration);
209 bufferPtrs[i] = out;
210 output[i].SetLength(outFrames);
211 }
212 MOZ_ASSERT(channels > 0);
213 c.mDuration = output[0].Length();
214 c.mBuffer = new mozilla::SharedChannelArrayBuffer<T>(&output);
215 for (uint32_t i = 0; i < channels; i++) {
216 c.mChannelData[i] = bufferPtrs[i];
217 }
218 mDuration += c.mDuration;
219 }
220 }
222 void ResampleChunks(SpeexResamplerState* aResampler);
224 void AppendFrames(already_AddRefed<ThreadSharedObject> aBuffer,
225 const nsTArray<const float*>& aChannelData,
226 int32_t aDuration)
227 {
228 AudioChunk* chunk = AppendChunk(aDuration);
229 chunk->mBuffer = aBuffer;
230 for (uint32_t channel = 0; channel < aChannelData.Length(); ++channel) {
231 chunk->mChannelData.AppendElement(aChannelData[channel]);
232 }
233 chunk->mVolume = 1.0f;
234 chunk->mBufferFormat = AUDIO_FORMAT_FLOAT32;
235 #ifdef MOZILLA_INTERNAL_API
236 chunk->mTimeStamp = TimeStamp::Now();
237 #endif
238 }
239 void AppendFrames(already_AddRefed<ThreadSharedObject> aBuffer,
240 const nsTArray<const int16_t*>& aChannelData,
241 int32_t aDuration)
242 {
243 AudioChunk* chunk = AppendChunk(aDuration);
244 chunk->mBuffer = aBuffer;
245 for (uint32_t channel = 0; channel < aChannelData.Length(); ++channel) {
246 chunk->mChannelData.AppendElement(aChannelData[channel]);
247 }
248 chunk->mVolume = 1.0f;
249 chunk->mBufferFormat = AUDIO_FORMAT_S16;
250 #ifdef MOZILLA_INTERNAL_API
251 chunk->mTimeStamp = TimeStamp::Now();
252 #endif
253 }
254 // Consumes aChunk, and returns a pointer to the persistent copy of aChunk
255 // in the segment.
256 AudioChunk* AppendAndConsumeChunk(AudioChunk* aChunk)
257 {
258 AudioChunk* chunk = AppendChunk(aChunk->mDuration);
259 chunk->mBuffer = aChunk->mBuffer.forget();
260 chunk->mChannelData.SwapElements(aChunk->mChannelData);
261 chunk->mVolume = aChunk->mVolume;
262 chunk->mBufferFormat = aChunk->mBufferFormat;
263 #ifdef MOZILLA_INTERNAL_API
264 chunk->mTimeStamp = TimeStamp::Now();
265 #endif
266 return chunk;
267 }
268 void ApplyVolume(float aVolume);
269 void WriteTo(uint64_t aID, AudioStream* aOutput, AudioMixer* aMixer = nullptr);
271 int ChannelCount() {
272 NS_WARN_IF_FALSE(!mChunks.IsEmpty(),
273 "Cannot query channel count on a AudioSegment with no chunks.");
274 // Find the first chunk that has non-zero channels. A chunk that hs zero
275 // channels is just silence and we can simply discard it.
276 for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
277 if (ci->ChannelCount()) {
278 return ci->ChannelCount();
279 }
280 }
281 return 0;
282 }
284 static Type StaticType() { return AUDIO; }
286 virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
287 {
288 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
289 }
290 };
292 }
294 #endif /* MOZILLA_AUDIOSEGMENT_H_ */
