content/media/mediasource/MediaSourceDecoder.cpp@97036ab72558
content/media/mediasource/MediaSourceDecoder.cpp
Tue, 06 Jan 2015 21:39:09 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Tue, 06 Jan 2015 21:39:09 +0100
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
- permissions
- -rw-r--r--
Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.
1 /* -*- mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "MediaSourceResource.h"
7 #include "MediaSourceDecoder.h"
9 #include "AbstractMediaDecoder.h"
10 #include "MediaDecoderReader.h"
11 #include "MediaDecoderStateMachine.h"
12 #include "mozilla/Assertions.h"
13 #include "mozilla/FloatingPoint.h"
14 #include "mozilla/dom/HTMLMediaElement.h"
15 #include "mozilla/dom/TimeRanges.h"
16 #include "mozilla/mozalloc.h"
17 #include "nsISupports.h"
18 #include "nsIThread.h"
19 #include "prlog.h"
20 #include "MediaSource.h"
21 #include "SubBufferDecoder.h"
22 #include "SourceBufferResource.h"
23 #include "SourceBufferList.h"
24 #include "VideoUtils.h"
26 #ifdef PR_LOGGING
27 extern PRLogModuleInfo* gMediaSourceLog;
28 #define MSE_DEBUG(...) PR_LOG(gMediaSourceLog, PR_LOG_DEBUG, (__VA_ARGS__))
29 #else
30 #define MSE_DEBUG(...)
31 #endif
33 namespace mozilla {
35 namespace dom {
37 class TimeRanges;
39 } // namespace dom
41 class MediaSourceReader : public MediaDecoderReader
42 {
43 public:
44 MediaSourceReader(MediaSourceDecoder* aDecoder, dom::MediaSource* aSource)
45 : MediaDecoderReader(aDecoder)
46 , mActiveVideoDecoder(-1)
47 , mActiveAudioDecoder(-1)
48 , mMediaSource(aSource)
49 {
50 }
52 nsresult Init(MediaDecoderReader* aCloneDonor) MOZ_OVERRIDE
53 {
54 // Although we technically don't implement anything here, we return NS_OK
55 // so that when the state machine initializes and calls this function
56 // we don't return an error code back to the media element.
57 return NS_OK;
58 }
60 bool IsWaitingMediaResources() MOZ_OVERRIDE
61 {
62 return mDecoders.IsEmpty() && mPendingDecoders.IsEmpty();
63 }
65 bool DecodeAudioData() MOZ_OVERRIDE
66 {
67 if (!GetAudioReader()) {
68 MSE_DEBUG("%p DecodeAudioFrame called with no audio reader", this);
69 MOZ_ASSERT(mPendingDecoders.IsEmpty());
70 return false;
71 }
72 bool rv = GetAudioReader()->DecodeAudioData();
74 nsAutoTArray<AudioData*, 10> audio;
75 GetAudioReader()->AudioQueue().GetElementsAfter(-1, &audio);
76 for (uint32_t i = 0; i < audio.Length(); ++i) {
77 AudioQueue().Push(audio[i]);
78 }
79 GetAudioReader()->AudioQueue().Empty();
81 return rv;
82 }
84 bool DecodeVideoFrame(bool& aKeyFrameSkip, int64_t aTimeThreshold) MOZ_OVERRIDE
85 {
86 if (!GetVideoReader()) {
87 MSE_DEBUG("%p DecodeVideoFrame called with no video reader", this);
88 MOZ_ASSERT(mPendingDecoders.IsEmpty());
89 return false;
90 }
91 bool rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
93 nsAutoTArray<VideoData*, 10> video;
94 GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
95 for (uint32_t i = 0; i < video.Length(); ++i) {
96 VideoQueue().Push(video[i]);
97 }
98 GetVideoReader()->VideoQueue().Empty();
100 if (rv) {
101 return true;
102 }
104 MSE_DEBUG("%p MSR::DecodeVF %d (%p) returned false (readers=%u)",
105 this, mActiveVideoDecoder, mDecoders[mActiveVideoDecoder].get(), mDecoders.Length());
106 if (SwitchVideoReaders(aTimeThreshold)) {
107 rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
109 nsAutoTArray<VideoData*, 10> video;
110 GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
111 for (uint32_t i = 0; i < video.Length(); ++i) {
112 VideoQueue().Push(video[i]);
113 }
114 GetVideoReader()->VideoQueue().Empty();
115 }
116 return rv;
117 }
119 bool HasVideo() MOZ_OVERRIDE
120 {
121 return mInfo.HasVideo();
122 }
124 bool HasAudio() MOZ_OVERRIDE
125 {
126 return mInfo.HasAudio();
127 }
129 nsresult ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) MOZ_OVERRIDE;
130 nsresult Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
131 int64_t aCurrentTime) MOZ_OVERRIDE;
132 nsresult GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime) MOZ_OVERRIDE;
133 already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
134 MediaSourceDecoder* aParentDecoder);
136 void CallDecoderInitialization();
138 private:
139 bool SwitchVideoReaders(int64_t aTimeThreshold) {
140 MOZ_ASSERT(mActiveVideoDecoder != -1);
141 // XXX: We switch when the first reader is depleted, but it might be
142 // better to switch as soon as the next reader is ready to decode and
143 // has data for the current media time.
144 ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
146 GetVideoReader()->SetIdle();
148 WaitForPendingDecoders();
150 for (uint32_t i = mActiveVideoDecoder + 1; i < mDecoders.Length(); ++i) {
151 if (!mDecoders[i]->GetReader()->GetMediaInfo().HasVideo()) {
152 continue;
153 }
154 mActiveVideoDecoder = i;
155 MSE_DEBUG("%p MSR::DecodeVF switching to %d", this, mActiveVideoDecoder);
157 GetVideoReader()->SetActive();
158 GetVideoReader()->DecodeToTarget(aTimeThreshold);
160 return true;
161 }
162 return false;
163 }
165 MediaDecoderReader* GetAudioReader() {
166 if (mActiveAudioDecoder == -1) {
167 return nullptr;
168 }
169 return mDecoders[mActiveAudioDecoder]->GetReader();
170 }
172 MediaDecoderReader* GetVideoReader() {
173 if (mActiveVideoDecoder == -1) {
174 return nullptr;
175 }
176 return mDecoders[mActiveVideoDecoder]->GetReader();
177 }
179 void WaitForPendingDecoders();
181 nsTArray<nsRefPtr<SubBufferDecoder>> mPendingDecoders;
182 nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
184 int32_t mActiveVideoDecoder;
185 int32_t mActiveAudioDecoder;
186 dom::MediaSource* mMediaSource;
187 };
189 class MediaSourceStateMachine : public MediaDecoderStateMachine
190 {
191 public:
192 MediaSourceStateMachine(MediaDecoder* aDecoder,
193 MediaDecoderReader* aReader,
194 bool aRealTime = false)
195 : MediaDecoderStateMachine(aDecoder, aReader, aRealTime)
196 {
197 }
199 already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
200 MediaSourceDecoder* aParentDecoder) {
201 if (!mReader) {
202 return nullptr;
203 }
204 return static_cast<MediaSourceReader*>(mReader.get())->CreateSubDecoder(aType, aParentDecoder);
205 }
207 nsresult EnqueueDecoderInitialization() {
208 AssertCurrentThreadInMonitor();
209 if (!mReader) {
210 return NS_ERROR_FAILURE;
211 }
212 return mDecodeTaskQueue->Dispatch(NS_NewRunnableMethod(this,
213 &MediaSourceStateMachine::CallDecoderInitialization));
214 }
216 private:
217 void CallDecoderInitialization() {
218 if (!mReader) {
219 return;
220 }
221 static_cast<MediaSourceReader*>(mReader.get())->CallDecoderInitialization();
222 }
223 };
225 MediaSourceDecoder::MediaSourceDecoder(dom::HTMLMediaElement* aElement)
226 : mMediaSource(nullptr)
227 {
228 Init(aElement);
229 }
231 MediaDecoder*
232 MediaSourceDecoder::Clone()
233 {
234 // TODO: Sort out cloning.
235 return nullptr;
236 }
238 MediaDecoderStateMachine*
239 MediaSourceDecoder::CreateStateMachine()
240 {
241 return new MediaSourceStateMachine(this, new MediaSourceReader(this, mMediaSource));
242 }
244 nsresult
245 MediaSourceDecoder::Load(nsIStreamListener**, MediaDecoder*)
246 {
247 MOZ_ASSERT(!mDecoderStateMachine);
248 mDecoderStateMachine = CreateStateMachine();
249 if (!mDecoderStateMachine) {
250 NS_WARNING("Failed to create state machine!");
251 return NS_ERROR_FAILURE;
252 }
254 return mDecoderStateMachine->Init(nullptr);
255 }
257 nsresult
258 MediaSourceDecoder::GetSeekable(dom::TimeRanges* aSeekable)
259 {
260 double duration = mMediaSource->Duration();
261 if (IsNaN(duration)) {
262 // Return empty range.
263 } else if (duration > 0 && mozilla::IsInfinite(duration)) {
264 nsRefPtr<dom::TimeRanges> bufferedRanges = new dom::TimeRanges();
265 GetBuffered(bufferedRanges);
266 aSeekable->Add(bufferedRanges->GetStartTime(), bufferedRanges->GetEndTime());
267 } else {
268 aSeekable->Add(0, duration);
269 }
270 return NS_OK;
271 }
273 /*static*/
274 already_AddRefed<MediaResource>
275 MediaSourceDecoder::CreateResource()
276 {
277 return nsRefPtr<MediaResource>(new MediaSourceResource()).forget();
278 }
280 void
281 MediaSourceDecoder::AttachMediaSource(dom::MediaSource* aMediaSource)
282 {
283 MOZ_ASSERT(!mMediaSource && !mDecoderStateMachine);
284 mMediaSource = aMediaSource;
285 }
287 void
288 MediaSourceDecoder::DetachMediaSource()
289 {
290 mMediaSource = nullptr;
291 }
293 already_AddRefed<SubBufferDecoder>
294 MediaSourceDecoder::CreateSubDecoder(const nsACString& aType)
295 {
296 if (!mDecoderStateMachine) {
297 return nullptr;
298 }
299 return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->CreateSubDecoder(aType, this);
300 }
302 nsresult
303 MediaSourceDecoder::EnqueueDecoderInitialization()
304 {
305 if (!mDecoderStateMachine) {
306 return NS_ERROR_FAILURE;
307 }
308 return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->EnqueueDecoderInitialization();
309 }
311 class ReleaseDecodersTask : public nsRunnable {
312 public:
313 ReleaseDecodersTask(nsTArray<nsRefPtr<SubBufferDecoder>>& aDecoders)
314 {
315 mDecoders.SwapElements(aDecoders);
316 }
318 NS_IMETHOD Run() MOZ_OVERRIDE MOZ_FINAL {
319 mDecoders.Clear();
320 return NS_OK;
321 }
323 private:
324 nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
325 };
327 void
328 MediaSourceReader::CallDecoderInitialization()
329 {
330 ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
331 for (uint32_t i = 0; i < mPendingDecoders.Length(); ++i) {
332 nsRefPtr<SubBufferDecoder> decoder = mPendingDecoders[i];
333 MediaDecoderReader* reader = decoder->GetReader();
334 MSE_DEBUG("%p: Initializating subdecoder %p reader %p", this, decoder.get(), reader);
336 reader->SetActive();
337 MediaInfo mi;
338 nsAutoPtr<MetadataTags> tags; // TODO: Handle metadata.
339 nsresult rv;
340 {
341 ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
342 rv = reader->ReadMetadata(&mi, getter_Transfers(tags));
343 }
344 reader->SetIdle();
345 if (NS_FAILED(rv)) {
346 // XXX: Need to signal error back to owning SourceBuffer.
347 MSE_DEBUG("%p: Reader %p failed to initialize, rv=%x", this, reader, rv);
348 continue;
349 }
351 bool active = false;
352 if (mi.HasVideo() || mi.HasAudio()) {
353 MSE_DEBUG("%p: Reader %p has video=%d audio=%d", this, reader, mi.HasVideo(), mi.HasAudio());
354 active = true;
355 }
357 if (active) {
358 mDecoders.AppendElement(decoder);
359 } else {
360 MSE_DEBUG("%p: Reader %p not activated", this, reader);
361 }
362 }
363 NS_DispatchToMainThread(new ReleaseDecodersTask(mPendingDecoders));
364 MOZ_ASSERT(mPendingDecoders.IsEmpty());
365 mDecoder->NotifyWaitingForResourcesStatusChanged();
366 mon.NotifyAll();
367 }
369 void
370 MediaSourceReader::WaitForPendingDecoders()
371 {
372 ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
373 while (!mPendingDecoders.IsEmpty()) {
374 mon.Wait();
375 }
376 }
378 already_AddRefed<SubBufferDecoder>
379 MediaSourceReader::CreateSubDecoder(const nsACString& aType, MediaSourceDecoder* aParentDecoder)
380 {
381 // XXX: Why/when is mDecoder null here, since it should be equal to aParentDecoder?!
382 nsRefPtr<SubBufferDecoder> decoder =
383 new SubBufferDecoder(new SourceBufferResource(nullptr, aType), aParentDecoder);
384 nsAutoPtr<MediaDecoderReader> reader(DecoderTraits::CreateReader(aType, decoder));
385 if (!reader) {
386 return nullptr;
387 }
388 reader->Init(nullptr);
389 ReentrantMonitorAutoEnter mon(aParentDecoder->GetReentrantMonitor());
390 MSE_DEBUG("Registered subdecoder %p subreader %p", decoder.get(), reader.get());
391 decoder->SetReader(reader.forget());
392 mPendingDecoders.AppendElement(decoder);
393 if (NS_FAILED(static_cast<MediaSourceDecoder*>(mDecoder)->EnqueueDecoderInitialization())) {
394 MSE_DEBUG("%p: Failed to enqueue decoder initialization task", this);
395 return nullptr;
396 }
397 mDecoder->NotifyWaitingForResourcesStatusChanged();
398 return decoder.forget();
399 }
401 nsresult
402 MediaSourceReader::Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
403 int64_t aCurrentTime)
404 {
405 ResetDecode();
407 dom::SourceBufferList* sbl = mMediaSource->ActiveSourceBuffers();
408 if (sbl->AllContainsTime (aTime / USECS_PER_S)) {
409 if (GetAudioReader()) {
410 nsresult rv = GetAudioReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
411 if (NS_FAILED(rv)) {
412 return rv;
413 }
414 }
415 if (GetVideoReader()) {
416 nsresult rv = GetVideoReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
417 if (NS_FAILED(rv)) {
418 return rv;
419 }
420 }
421 }
422 return NS_OK;
423 }
425 nsresult
426 MediaSourceReader::GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime)
427 {
428 for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
429 nsRefPtr<dom::TimeRanges> r = new dom::TimeRanges();
430 mDecoders[i]->GetBuffered(r);
431 aBuffered->Add(r->GetStartTime(), r->GetEndTime());
432 }
433 aBuffered->Normalize();
434 return NS_OK;
435 }
437 nsresult
438 MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags)
439 {
440 mDecoder->SetMediaSeekable(true);
441 mDecoder->SetTransportSeekable(false);
443 MSE_DEBUG("%p: MSR::ReadMetadata pending=%u", this, mPendingDecoders.Length());
445 WaitForPendingDecoders();
447 MSE_DEBUG("%p: MSR::ReadMetadata decoders=%u", this, mDecoders.Length());
449 // XXX: Make subdecoder setup async, so that use cases like bug 989888 can
450 // work. This will require teaching the state machine about dynamic track
451 // changes (and multiple tracks).
452 // Shorter term, make this block until we've got at least one video track
453 // and lie about having an audio track, then resample/remix as necessary
454 // to match any audio track added later to fit the format we lied about
455 // now. For now we just configure what we've got and cross our fingers.
456 int64_t maxDuration = -1;
457 for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
458 MediaDecoderReader* reader = mDecoders[i]->GetReader();
460 reader->SetActive(); // XXX check where this should be called
462 MediaInfo mi = reader->GetMediaInfo();
464 if (mi.HasVideo() && !mInfo.HasVideo()) {
465 MOZ_ASSERT(mActiveVideoDecoder == -1);
466 mActiveVideoDecoder = i;
467 mInfo.mVideo = mi.mVideo;
468 maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
469 MSE_DEBUG("%p: MSR::ReadMetadata video decoder=%u maxDuration=%lld", this, i, maxDuration);
470 }
471 if (mi.HasAudio() && !mInfo.HasAudio()) {
472 MOZ_ASSERT(mActiveAudioDecoder == -1);
473 mActiveAudioDecoder = i;
474 mInfo.mAudio = mi.mAudio;
475 maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
476 MSE_DEBUG("%p: MSR::ReadMetadata audio decoder=%u maxDuration=%lld", this, i, maxDuration);
477 }
478 }
480 if (maxDuration != -1) {
481 ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
482 mDecoder->SetMediaDuration(maxDuration);
483 ErrorResult dummy;
484 mMediaSource->SetDuration(maxDuration, dummy);
485 }
487 *aInfo = mInfo;
488 *aTags = nullptr; // TODO: Handle metadata.
490 return NS_OK;
491 }
493 } // namespace mozilla
