1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/content/media/mediasource/MediaSourceDecoder.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,493 @@
1.4 +/* -*- mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
1.5 +/* vim: set ts=8 sts=2 et sw=2 tw=80: */
1.6 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.7 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.9 +#include "MediaSourceResource.h"
1.10 +#include "MediaSourceDecoder.h"
1.11 +
1.12 +#include "AbstractMediaDecoder.h"
1.13 +#include "MediaDecoderReader.h"
1.14 +#include "MediaDecoderStateMachine.h"
1.15 +#include "mozilla/Assertions.h"
1.16 +#include "mozilla/FloatingPoint.h"
1.17 +#include "mozilla/dom/HTMLMediaElement.h"
1.18 +#include "mozilla/dom/TimeRanges.h"
1.19 +#include "mozilla/mozalloc.h"
1.20 +#include "nsISupports.h"
1.21 +#include "nsIThread.h"
1.22 +#include "prlog.h"
1.23 +#include "MediaSource.h"
1.24 +#include "SubBufferDecoder.h"
1.25 +#include "SourceBufferResource.h"
1.26 +#include "SourceBufferList.h"
1.27 +#include "VideoUtils.h"
1.28 +
1.29 +#ifdef PR_LOGGING
1.30 +extern PRLogModuleInfo* gMediaSourceLog;
1.31 +#define MSE_DEBUG(...) PR_LOG(gMediaSourceLog, PR_LOG_DEBUG, (__VA_ARGS__))
1.32 +#else
1.33 +#define MSE_DEBUG(...)
1.34 +#endif
1.35 +
1.36 +namespace mozilla {
1.37 +
1.38 +namespace dom {
1.39 +
1.40 +class TimeRanges;
1.41 +
1.42 +} // namespace dom
1.43 +
1.44 +class MediaSourceReader : public MediaDecoderReader
1.45 +{
1.46 +public:
1.47 + MediaSourceReader(MediaSourceDecoder* aDecoder, dom::MediaSource* aSource)
1.48 + : MediaDecoderReader(aDecoder)
1.49 + , mActiveVideoDecoder(-1)
1.50 + , mActiveAudioDecoder(-1)
1.51 + , mMediaSource(aSource)
1.52 + {
1.53 + }
1.54 +
1.55 + nsresult Init(MediaDecoderReader* aCloneDonor) MOZ_OVERRIDE
1.56 + {
1.57 + // Although we technically don't implement anything here, we return NS_OK
1.58 + // so that when the state machine initializes and calls this function
1.59 + // we don't return an error code back to the media element.
1.60 + return NS_OK;
1.61 + }
1.62 +
1.63 + bool IsWaitingMediaResources() MOZ_OVERRIDE
1.64 + {
1.65 + return mDecoders.IsEmpty() && mPendingDecoders.IsEmpty();
1.66 + }
1.67 +
1.68 + bool DecodeAudioData() MOZ_OVERRIDE
1.69 + {
1.70 + if (!GetAudioReader()) {
1.71 + MSE_DEBUG("%p DecodeAudioFrame called with no audio reader", this);
1.72 + MOZ_ASSERT(mPendingDecoders.IsEmpty());
1.73 + return false;
1.74 + }
1.75 + bool rv = GetAudioReader()->DecodeAudioData();
1.76 +
1.77 + nsAutoTArray<AudioData*, 10> audio;
1.78 + GetAudioReader()->AudioQueue().GetElementsAfter(-1, &audio);
1.79 + for (uint32_t i = 0; i < audio.Length(); ++i) {
1.80 + AudioQueue().Push(audio[i]);
1.81 + }
1.82 + GetAudioReader()->AudioQueue().Empty();
1.83 +
1.84 + return rv;
1.85 + }
1.86 +
1.87 + bool DecodeVideoFrame(bool& aKeyFrameSkip, int64_t aTimeThreshold) MOZ_OVERRIDE
1.88 + {
1.89 + if (!GetVideoReader()) {
1.90 + MSE_DEBUG("%p DecodeVideoFrame called with no video reader", this);
1.91 + MOZ_ASSERT(mPendingDecoders.IsEmpty());
1.92 + return false;
1.93 + }
1.94 + bool rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
1.95 +
1.96 + nsAutoTArray<VideoData*, 10> video;
1.97 + GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
1.98 + for (uint32_t i = 0; i < video.Length(); ++i) {
1.99 + VideoQueue().Push(video[i]);
1.100 + }
1.101 + GetVideoReader()->VideoQueue().Empty();
1.102 +
1.103 + if (rv) {
1.104 + return true;
1.105 + }
1.106 +
1.107 + MSE_DEBUG("%p MSR::DecodeVF %d (%p) returned false (readers=%u)",
1.108 + this, mActiveVideoDecoder, mDecoders[mActiveVideoDecoder].get(), mDecoders.Length());
1.109 + if (SwitchVideoReaders(aTimeThreshold)) {
1.110 + rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
1.111 +
1.112 + nsAutoTArray<VideoData*, 10> video;
1.113 + GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
1.114 + for (uint32_t i = 0; i < video.Length(); ++i) {
1.115 + VideoQueue().Push(video[i]);
1.116 + }
1.117 + GetVideoReader()->VideoQueue().Empty();
1.118 + }
1.119 + return rv;
1.120 + }
1.121 +
1.122 + bool HasVideo() MOZ_OVERRIDE
1.123 + {
1.124 + return mInfo.HasVideo();
1.125 + }
1.126 +
1.127 + bool HasAudio() MOZ_OVERRIDE
1.128 + {
1.129 + return mInfo.HasAudio();
1.130 + }
1.131 +
1.132 + nsresult ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) MOZ_OVERRIDE;
1.133 + nsresult Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
1.134 + int64_t aCurrentTime) MOZ_OVERRIDE;
1.135 + nsresult GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime) MOZ_OVERRIDE;
1.136 + already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
1.137 + MediaSourceDecoder* aParentDecoder);
1.138 +
1.139 + void CallDecoderInitialization();
1.140 +
1.141 +private:
1.142 + bool SwitchVideoReaders(int64_t aTimeThreshold) {
1.143 + MOZ_ASSERT(mActiveVideoDecoder != -1);
1.144 + // XXX: We switch when the first reader is depleted, but it might be
1.145 + // better to switch as soon as the next reader is ready to decode and
1.146 + // has data for the current media time.
1.147 + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
1.148 +
1.149 + GetVideoReader()->SetIdle();
1.150 +
1.151 + WaitForPendingDecoders();
1.152 +
1.153 + for (uint32_t i = mActiveVideoDecoder + 1; i < mDecoders.Length(); ++i) {
1.154 + if (!mDecoders[i]->GetReader()->GetMediaInfo().HasVideo()) {
1.155 + continue;
1.156 + }
1.157 + mActiveVideoDecoder = i;
1.158 + MSE_DEBUG("%p MSR::DecodeVF switching to %d", this, mActiveVideoDecoder);
1.159 +
1.160 + GetVideoReader()->SetActive();
1.161 + GetVideoReader()->DecodeToTarget(aTimeThreshold);
1.162 +
1.163 + return true;
1.164 + }
1.165 + return false;
1.166 + }
1.167 +
1.168 + MediaDecoderReader* GetAudioReader() {
1.169 + if (mActiveAudioDecoder == -1) {
1.170 + return nullptr;
1.171 + }
1.172 + return mDecoders[mActiveAudioDecoder]->GetReader();
1.173 + }
1.174 +
1.175 + MediaDecoderReader* GetVideoReader() {
1.176 + if (mActiveVideoDecoder == -1) {
1.177 + return nullptr;
1.178 + }
1.179 + return mDecoders[mActiveVideoDecoder]->GetReader();
1.180 + }
1.181 +
1.182 + void WaitForPendingDecoders();
1.183 +
1.184 + nsTArray<nsRefPtr<SubBufferDecoder>> mPendingDecoders;
1.185 + nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
1.186 +
1.187 + int32_t mActiveVideoDecoder;
1.188 + int32_t mActiveAudioDecoder;
1.189 + dom::MediaSource* mMediaSource;
1.190 +};
1.191 +
1.192 +class MediaSourceStateMachine : public MediaDecoderStateMachine
1.193 +{
1.194 +public:
1.195 + MediaSourceStateMachine(MediaDecoder* aDecoder,
1.196 + MediaDecoderReader* aReader,
1.197 + bool aRealTime = false)
1.198 + : MediaDecoderStateMachine(aDecoder, aReader, aRealTime)
1.199 + {
1.200 + }
1.201 +
1.202 + already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
1.203 + MediaSourceDecoder* aParentDecoder) {
1.204 + if (!mReader) {
1.205 + return nullptr;
1.206 + }
1.207 + return static_cast<MediaSourceReader*>(mReader.get())->CreateSubDecoder(aType, aParentDecoder);
1.208 + }
1.209 +
1.210 + nsresult EnqueueDecoderInitialization() {
1.211 + AssertCurrentThreadInMonitor();
1.212 + if (!mReader) {
1.213 + return NS_ERROR_FAILURE;
1.214 + }
1.215 + return mDecodeTaskQueue->Dispatch(NS_NewRunnableMethod(this,
1.216 + &MediaSourceStateMachine::CallDecoderInitialization));
1.217 + }
1.218 +
1.219 +private:
1.220 + void CallDecoderInitialization() {
1.221 + if (!mReader) {
1.222 + return;
1.223 + }
1.224 + static_cast<MediaSourceReader*>(mReader.get())->CallDecoderInitialization();
1.225 + }
1.226 +};
1.227 +
1.228 +MediaSourceDecoder::MediaSourceDecoder(dom::HTMLMediaElement* aElement)
1.229 + : mMediaSource(nullptr)
1.230 +{
1.231 + Init(aElement);
1.232 +}
1.233 +
1.234 +MediaDecoder*
1.235 +MediaSourceDecoder::Clone()
1.236 +{
1.237 + // TODO: Sort out cloning.
1.238 + return nullptr;
1.239 +}
1.240 +
1.241 +MediaDecoderStateMachine*
1.242 +MediaSourceDecoder::CreateStateMachine()
1.243 +{
1.244 + return new MediaSourceStateMachine(this, new MediaSourceReader(this, mMediaSource));
1.245 +}
1.246 +
1.247 +nsresult
1.248 +MediaSourceDecoder::Load(nsIStreamListener**, MediaDecoder*)
1.249 +{
1.250 + MOZ_ASSERT(!mDecoderStateMachine);
1.251 + mDecoderStateMachine = CreateStateMachine();
1.252 + if (!mDecoderStateMachine) {
1.253 + NS_WARNING("Failed to create state machine!");
1.254 + return NS_ERROR_FAILURE;
1.255 + }
1.256 +
1.257 + return mDecoderStateMachine->Init(nullptr);
1.258 +}
1.259 +
1.260 +nsresult
1.261 +MediaSourceDecoder::GetSeekable(dom::TimeRanges* aSeekable)
1.262 +{
1.263 + double duration = mMediaSource->Duration();
1.264 + if (IsNaN(duration)) {
1.265 + // Return empty range.
1.266 + } else if (duration > 0 && mozilla::IsInfinite(duration)) {
1.267 + nsRefPtr<dom::TimeRanges> bufferedRanges = new dom::TimeRanges();
1.268 + GetBuffered(bufferedRanges);
1.269 + aSeekable->Add(bufferedRanges->GetStartTime(), bufferedRanges->GetEndTime());
1.270 + } else {
1.271 + aSeekable->Add(0, duration);
1.272 + }
1.273 + return NS_OK;
1.274 +}
1.275 +
1.276 +/*static*/
1.277 +already_AddRefed<MediaResource>
1.278 +MediaSourceDecoder::CreateResource()
1.279 +{
1.280 + return nsRefPtr<MediaResource>(new MediaSourceResource()).forget();
1.281 +}
1.282 +
1.283 +void
1.284 +MediaSourceDecoder::AttachMediaSource(dom::MediaSource* aMediaSource)
1.285 +{
1.286 + MOZ_ASSERT(!mMediaSource && !mDecoderStateMachine);
1.287 + mMediaSource = aMediaSource;
1.288 +}
1.289 +
1.290 +void
1.291 +MediaSourceDecoder::DetachMediaSource()
1.292 +{
1.293 + mMediaSource = nullptr;
1.294 +}
1.295 +
1.296 +already_AddRefed<SubBufferDecoder>
1.297 +MediaSourceDecoder::CreateSubDecoder(const nsACString& aType)
1.298 +{
1.299 + if (!mDecoderStateMachine) {
1.300 + return nullptr;
1.301 + }
1.302 + return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->CreateSubDecoder(aType, this);
1.303 +}
1.304 +
1.305 +nsresult
1.306 +MediaSourceDecoder::EnqueueDecoderInitialization()
1.307 +{
1.308 + if (!mDecoderStateMachine) {
1.309 + return NS_ERROR_FAILURE;
1.310 + }
1.311 + return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->EnqueueDecoderInitialization();
1.312 +}
1.313 +
1.314 +class ReleaseDecodersTask : public nsRunnable {
1.315 +public:
1.316 + ReleaseDecodersTask(nsTArray<nsRefPtr<SubBufferDecoder>>& aDecoders)
1.317 + {
1.318 + mDecoders.SwapElements(aDecoders);
1.319 + }
1.320 +
1.321 + NS_IMETHOD Run() MOZ_OVERRIDE MOZ_FINAL {
1.322 + mDecoders.Clear();
1.323 + return NS_OK;
1.324 + }
1.325 +
1.326 +private:
1.327 + nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
1.328 +};
1.329 +
1.330 +void
1.331 +MediaSourceReader::CallDecoderInitialization()
1.332 +{
1.333 + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
1.334 + for (uint32_t i = 0; i < mPendingDecoders.Length(); ++i) {
1.335 + nsRefPtr<SubBufferDecoder> decoder = mPendingDecoders[i];
1.336 + MediaDecoderReader* reader = decoder->GetReader();
1.337 + MSE_DEBUG("%p: Initializating subdecoder %p reader %p", this, decoder.get(), reader);
1.338 +
1.339 + reader->SetActive();
1.340 + MediaInfo mi;
1.341 + nsAutoPtr<MetadataTags> tags; // TODO: Handle metadata.
1.342 + nsresult rv;
1.343 + {
1.344 + ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
1.345 + rv = reader->ReadMetadata(&mi, getter_Transfers(tags));
1.346 + }
1.347 + reader->SetIdle();
1.348 + if (NS_FAILED(rv)) {
1.349 + // XXX: Need to signal error back to owning SourceBuffer.
1.350 + MSE_DEBUG("%p: Reader %p failed to initialize, rv=%x", this, reader, rv);
1.351 + continue;
1.352 + }
1.353 +
1.354 + bool active = false;
1.355 + if (mi.HasVideo() || mi.HasAudio()) {
1.356 + MSE_DEBUG("%p: Reader %p has video=%d audio=%d", this, reader, mi.HasVideo(), mi.HasAudio());
1.357 + active = true;
1.358 + }
1.359 +
1.360 + if (active) {
1.361 + mDecoders.AppendElement(decoder);
1.362 + } else {
1.363 + MSE_DEBUG("%p: Reader %p not activated", this, reader);
1.364 + }
1.365 + }
1.366 + NS_DispatchToMainThread(new ReleaseDecodersTask(mPendingDecoders));
1.367 + MOZ_ASSERT(mPendingDecoders.IsEmpty());
1.368 + mDecoder->NotifyWaitingForResourcesStatusChanged();
1.369 + mon.NotifyAll();
1.370 +}
1.371 +
1.372 +void
1.373 +MediaSourceReader::WaitForPendingDecoders()
1.374 +{
1.375 + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
1.376 + while (!mPendingDecoders.IsEmpty()) {
1.377 + mon.Wait();
1.378 + }
1.379 +}
1.380 +
1.381 +already_AddRefed<SubBufferDecoder>
1.382 +MediaSourceReader::CreateSubDecoder(const nsACString& aType, MediaSourceDecoder* aParentDecoder)
1.383 +{
1.384 + // XXX: Why/when is mDecoder null here, since it should be equal to aParentDecoder?!
1.385 + nsRefPtr<SubBufferDecoder> decoder =
1.386 + new SubBufferDecoder(new SourceBufferResource(nullptr, aType), aParentDecoder);
1.387 + nsAutoPtr<MediaDecoderReader> reader(DecoderTraits::CreateReader(aType, decoder));
1.388 + if (!reader) {
1.389 + return nullptr;
1.390 + }
1.391 + reader->Init(nullptr);
1.392 + ReentrantMonitorAutoEnter mon(aParentDecoder->GetReentrantMonitor());
1.393 + MSE_DEBUG("Registered subdecoder %p subreader %p", decoder.get(), reader.get());
1.394 + decoder->SetReader(reader.forget());
1.395 + mPendingDecoders.AppendElement(decoder);
1.396 + if (NS_FAILED(static_cast<MediaSourceDecoder*>(mDecoder)->EnqueueDecoderInitialization())) {
1.397 + MSE_DEBUG("%p: Failed to enqueue decoder initialization task", this);
1.398 + return nullptr;
1.399 + }
1.400 + mDecoder->NotifyWaitingForResourcesStatusChanged();
1.401 + return decoder.forget();
1.402 +}
1.403 +
1.404 +nsresult
1.405 +MediaSourceReader::Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
1.406 + int64_t aCurrentTime)
1.407 +{
1.408 + ResetDecode();
1.409 +
1.410 + dom::SourceBufferList* sbl = mMediaSource->ActiveSourceBuffers();
1.411 + if (sbl->AllContainsTime (aTime / USECS_PER_S)) {
1.412 + if (GetAudioReader()) {
1.413 + nsresult rv = GetAudioReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
1.414 + if (NS_FAILED(rv)) {
1.415 + return rv;
1.416 + }
1.417 + }
1.418 + if (GetVideoReader()) {
1.419 + nsresult rv = GetVideoReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
1.420 + if (NS_FAILED(rv)) {
1.421 + return rv;
1.422 + }
1.423 + }
1.424 + }
1.425 + return NS_OK;
1.426 +}
1.427 +
1.428 +nsresult
1.429 +MediaSourceReader::GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime)
1.430 +{
1.431 + for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
1.432 + nsRefPtr<dom::TimeRanges> r = new dom::TimeRanges();
1.433 + mDecoders[i]->GetBuffered(r);
1.434 + aBuffered->Add(r->GetStartTime(), r->GetEndTime());
1.435 + }
1.436 + aBuffered->Normalize();
1.437 + return NS_OK;
1.438 +}
1.439 +
1.440 +nsresult
1.441 +MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags)
1.442 +{
1.443 + mDecoder->SetMediaSeekable(true);
1.444 + mDecoder->SetTransportSeekable(false);
1.445 +
1.446 + MSE_DEBUG("%p: MSR::ReadMetadata pending=%u", this, mPendingDecoders.Length());
1.447 +
1.448 + WaitForPendingDecoders();
1.449 +
1.450 + MSE_DEBUG("%p: MSR::ReadMetadata decoders=%u", this, mDecoders.Length());
1.451 +
1.452 + // XXX: Make subdecoder setup async, so that use cases like bug 989888 can
1.453 + // work. This will require teaching the state machine about dynamic track
1.454 + // changes (and multiple tracks).
1.455 + // Shorter term, make this block until we've got at least one video track
1.456 + // and lie about having an audio track, then resample/remix as necessary
1.457 + // to match any audio track added later to fit the format we lied about
1.458 + // now. For now we just configure what we've got and cross our fingers.
1.459 + int64_t maxDuration = -1;
1.460 + for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
1.461 + MediaDecoderReader* reader = mDecoders[i]->GetReader();
1.462 +
1.463 + reader->SetActive(); // XXX check where this should be called
1.464 +
1.465 + MediaInfo mi = reader->GetMediaInfo();
1.466 +
1.467 + if (mi.HasVideo() && !mInfo.HasVideo()) {
1.468 + MOZ_ASSERT(mActiveVideoDecoder == -1);
1.469 + mActiveVideoDecoder = i;
1.470 + mInfo.mVideo = mi.mVideo;
1.471 + maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
1.472 + MSE_DEBUG("%p: MSR::ReadMetadata video decoder=%u maxDuration=%lld", this, i, maxDuration);
1.473 + }
1.474 + if (mi.HasAudio() && !mInfo.HasAudio()) {
1.475 + MOZ_ASSERT(mActiveAudioDecoder == -1);
1.476 + mActiveAudioDecoder = i;
1.477 + mInfo.mAudio = mi.mAudio;
1.478 + maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
1.479 + MSE_DEBUG("%p: MSR::ReadMetadata audio decoder=%u maxDuration=%lld", this, i, maxDuration);
1.480 + }
1.481 + }
1.482 +
1.483 + if (maxDuration != -1) {
1.484 + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
1.485 + mDecoder->SetMediaDuration(maxDuration);
1.486 + ErrorResult dummy;
1.487 + mMediaSource->SetDuration(maxDuration, dummy);
1.488 + }
1.489 +
1.490 + *aInfo = mInfo;
1.491 + *aTags = nullptr; // TODO: Handle metadata.
1.492 +
1.493 + return NS_OK;
1.494 +}
1.495 +
1.496 +} // namespace mozilla
