michael@0: /* -*- mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* vim: set ts=8 sts=2 et sw=2 tw=80: */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: #include "MediaSourceResource.h"
michael@0: #include "MediaSourceDecoder.h"
michael@0: 
michael@0: #include "AbstractMediaDecoder.h"
michael@0: #include "MediaDecoderReader.h"
michael@0: #include "MediaDecoderStateMachine.h"
michael@0: #include "mozilla/Assertions.h"
michael@0: #include "mozilla/FloatingPoint.h"
michael@0: #include "mozilla/dom/HTMLMediaElement.h"
michael@0: #include "mozilla/dom/TimeRanges.h"
michael@0: #include "mozilla/mozalloc.h"
michael@0: #include "nsISupports.h"
michael@0: #include "nsIThread.h"
michael@0: #include "prlog.h"
michael@0: #include "MediaSource.h"
michael@0: #include "SubBufferDecoder.h"
michael@0: #include "SourceBufferResource.h"
michael@0: #include "SourceBufferList.h"
michael@0: #include "VideoUtils.h"
michael@0: 
michael@0: #ifdef PR_LOGGING
michael@0: extern PRLogModuleInfo* gMediaSourceLog;
michael@0: #define MSE_DEBUG(...) PR_LOG(gMediaSourceLog, PR_LOG_DEBUG, (__VA_ARGS__))
michael@0: #else
michael@0: #define MSE_DEBUG(...)
michael@0: #endif
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: namespace dom {
michael@0: 
michael@0: class TimeRanges;
michael@0: 
michael@0: } // namespace dom
michael@0: 
michael@0: class MediaSourceReader : public MediaDecoderReader
michael@0: {
michael@0: public:
michael@0:   MediaSourceReader(MediaSourceDecoder* aDecoder, dom::MediaSource* aSource)
michael@0:     : MediaDecoderReader(aDecoder)
michael@0:     , mActiveVideoDecoder(-1)
michael@0:     , mActiveAudioDecoder(-1)
michael@0:     , mMediaSource(aSource)
michael@0:   {
michael@0:   }
michael@0: 
michael@0:   nsresult Init(MediaDecoderReader* aCloneDonor) MOZ_OVERRIDE
michael@0:   {
michael@0:     // Although we technically don't implement anything here, we return NS_OK
michael@0:     // so that when the state machine initializes and calls this function
michael@0:     // we don't return an error code back to the media element.
michael@0:     return NS_OK;
michael@0:   }
michael@0: 
michael@0:   bool IsWaitingMediaResources() MOZ_OVERRIDE
michael@0:   {
michael@0:     return mDecoders.IsEmpty() && mPendingDecoders.IsEmpty();
michael@0:   }
michael@0: 
michael@0:   bool DecodeAudioData() MOZ_OVERRIDE
michael@0:   {
michael@0:     if (!GetAudioReader()) {
michael@0:       MSE_DEBUG("%p DecodeAudioFrame called with no audio reader", this);
michael@0:       MOZ_ASSERT(mPendingDecoders.IsEmpty());
michael@0:       return false;
michael@0:     }
michael@0:     bool rv = GetAudioReader()->DecodeAudioData();
michael@0: 
michael@0:     nsAutoTArray<AudioData*, 10> audio;
michael@0:     GetAudioReader()->AudioQueue().GetElementsAfter(-1, &audio);
michael@0:     for (uint32_t i = 0; i < audio.Length(); ++i) {
michael@0:       AudioQueue().Push(audio[i]);
michael@0:     }
michael@0:     GetAudioReader()->AudioQueue().Empty();
michael@0: 
michael@0:     return rv;
michael@0:   }
michael@0: 
michael@0:   bool DecodeVideoFrame(bool& aKeyFrameSkip, int64_t aTimeThreshold) MOZ_OVERRIDE
michael@0:   {
michael@0:     if (!GetVideoReader()) {
michael@0:       MSE_DEBUG("%p DecodeVideoFrame called with no video reader", this);
michael@0:       MOZ_ASSERT(mPendingDecoders.IsEmpty());
michael@0:       return false;
michael@0:     }
michael@0:     bool rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
michael@0: 
michael@0:     nsAutoTArray<VideoData*, 10> video;
michael@0:     GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
michael@0:     for (uint32_t i = 0; i < video.Length(); ++i) {
michael@0:       VideoQueue().Push(video[i]);
michael@0:     }
michael@0:     GetVideoReader()->VideoQueue().Empty();
michael@0: 
michael@0:     if (rv) {
michael@0:       return true;
michael@0:     }
michael@0: 
michael@0:     MSE_DEBUG("%p MSR::DecodeVF %d (%p) returned false (readers=%u)",
michael@0:               this, mActiveVideoDecoder, mDecoders[mActiveVideoDecoder].get(), mDecoders.Length());
michael@0:     if (SwitchVideoReaders(aTimeThreshold)) {
michael@0:       rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
michael@0: 
michael@0:       nsAutoTArray<VideoData*, 10> video;
michael@0:       GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
michael@0:       for (uint32_t i = 0; i < video.Length(); ++i) {
michael@0:         VideoQueue().Push(video[i]);
michael@0:       }
michael@0:       GetVideoReader()->VideoQueue().Empty();
michael@0:     }
michael@0:     return rv;
michael@0:   }
michael@0: 
michael@0:   bool HasVideo() MOZ_OVERRIDE
michael@0:   {
michael@0:     return mInfo.HasVideo();
michael@0:   }
michael@0: 
michael@0:   bool HasAudio() MOZ_OVERRIDE
michael@0:   {
michael@0:     return mInfo.HasAudio();
michael@0:   }
michael@0: 
michael@0:   nsresult ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) MOZ_OVERRIDE;
michael@0:   nsresult Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
michael@0:                 int64_t aCurrentTime) MOZ_OVERRIDE;
michael@0:   nsresult GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime) MOZ_OVERRIDE;
michael@0:   already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
michael@0:                                                       MediaSourceDecoder* aParentDecoder);
michael@0: 
michael@0:   void CallDecoderInitialization();
michael@0: 
michael@0: private:
michael@0:   bool SwitchVideoReaders(int64_t aTimeThreshold) {
michael@0:     MOZ_ASSERT(mActiveVideoDecoder != -1);
michael@0:     // XXX: We switch when the first reader is depleted, but it might be
michael@0:     // better to switch as soon as the next reader is ready to decode and
michael@0:     // has data for the current media time.
michael@0:     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
michael@0: 
michael@0:     GetVideoReader()->SetIdle();
michael@0: 
michael@0:     WaitForPendingDecoders();
michael@0: 
michael@0:     for (uint32_t i = mActiveVideoDecoder + 1; i < mDecoders.Length(); ++i) {
michael@0:       if (!mDecoders[i]->GetReader()->GetMediaInfo().HasVideo()) {
michael@0:         continue;
michael@0:       }
michael@0:       mActiveVideoDecoder = i;
michael@0:       MSE_DEBUG("%p MSR::DecodeVF switching to %d", this, mActiveVideoDecoder);
michael@0: 
michael@0:       GetVideoReader()->SetActive();
michael@0:       GetVideoReader()->DecodeToTarget(aTimeThreshold);
michael@0: 
michael@0:       return true;
michael@0:     }
michael@0:     return false;
michael@0:   }
michael@0: 
michael@0:   MediaDecoderReader* GetAudioReader() {
michael@0:     if (mActiveAudioDecoder == -1) {
michael@0:       return nullptr;
michael@0:     }
michael@0:     return mDecoders[mActiveAudioDecoder]->GetReader();
michael@0:   }
michael@0: 
michael@0:   MediaDecoderReader* GetVideoReader() {
michael@0:     if (mActiveVideoDecoder == -1) {
michael@0:       return nullptr;
michael@0:     }
michael@0:     return mDecoders[mActiveVideoDecoder]->GetReader();
michael@0:   }
michael@0: 
michael@0:   void WaitForPendingDecoders();
michael@0: 
michael@0:   nsTArray<nsRefPtr<SubBufferDecoder>> mPendingDecoders;
michael@0:   nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
michael@0: 
michael@0:   int32_t mActiveVideoDecoder;
michael@0:   int32_t mActiveAudioDecoder;
michael@0:   dom::MediaSource* mMediaSource;
michael@0: };
michael@0: 
michael@0: class MediaSourceStateMachine : public MediaDecoderStateMachine
michael@0: {
michael@0: public:
michael@0:   MediaSourceStateMachine(MediaDecoder* aDecoder,
michael@0:                           MediaDecoderReader* aReader,
michael@0:                           bool aRealTime = false)
michael@0:     : MediaDecoderStateMachine(aDecoder, aReader, aRealTime)
michael@0:   {
michael@0:   }
michael@0: 
michael@0:   already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
michael@0:                                                       MediaSourceDecoder* aParentDecoder) {
michael@0:     if (!mReader) {
michael@0:       return nullptr;
michael@0:     }
michael@0:     return static_cast<MediaSourceReader*>(mReader.get())->CreateSubDecoder(aType, aParentDecoder);
michael@0:   }
michael@0: 
michael@0:   nsresult EnqueueDecoderInitialization() {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     if (!mReader) {
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0:     return mDecodeTaskQueue->Dispatch(NS_NewRunnableMethod(this,
michael@0:                                                            &MediaSourceStateMachine::CallDecoderInitialization));
michael@0:   }
michael@0: 
michael@0: private:
michael@0:   void CallDecoderInitialization() {
michael@0:     if (!mReader) {
michael@0:       return;
michael@0:     }
michael@0:     static_cast<MediaSourceReader*>(mReader.get())->CallDecoderInitialization();
michael@0:   }
michael@0: };
michael@0: 
michael@0: MediaSourceDecoder::MediaSourceDecoder(dom::HTMLMediaElement* aElement)
michael@0:   : mMediaSource(nullptr)
michael@0: {
michael@0:   Init(aElement);
michael@0: }
michael@0: 
michael@0: MediaDecoder*
michael@0: MediaSourceDecoder::Clone()
michael@0: {
michael@0:   // TODO: Sort out cloning.
michael@0:   return nullptr;
michael@0: }
michael@0: 
michael@0: MediaDecoderStateMachine*
michael@0: MediaSourceDecoder::CreateStateMachine()
michael@0: {
michael@0:   return new MediaSourceStateMachine(this, new MediaSourceReader(this, mMediaSource));
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaSourceDecoder::Load(nsIStreamListener**, MediaDecoder*)
michael@0: {
michael@0:   MOZ_ASSERT(!mDecoderStateMachine);
michael@0:   mDecoderStateMachine = CreateStateMachine();
michael@0:   if (!mDecoderStateMachine) {
michael@0:     NS_WARNING("Failed to create state machine!");
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   return mDecoderStateMachine->Init(nullptr);
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaSourceDecoder::GetSeekable(dom::TimeRanges* aSeekable)
michael@0: {
michael@0:   double duration = mMediaSource->Duration();
michael@0:   if (IsNaN(duration)) {
michael@0:     // Return empty range.
michael@0:   } else if (duration > 0 && mozilla::IsInfinite(duration)) {
michael@0:     nsRefPtr<dom::TimeRanges> bufferedRanges = new dom::TimeRanges();
michael@0:     GetBuffered(bufferedRanges);
michael@0:     aSeekable->Add(bufferedRanges->GetStartTime(), bufferedRanges->GetEndTime());
michael@0:   } else {
michael@0:     aSeekable->Add(0, duration);
michael@0:   }
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: /*static*/
michael@0: already_AddRefed<MediaResource>
michael@0: MediaSourceDecoder::CreateResource()
michael@0: {
michael@0:   return nsRefPtr<MediaResource>(new MediaSourceResource()).forget();
michael@0: }
michael@0: 
michael@0: void
michael@0: MediaSourceDecoder::AttachMediaSource(dom::MediaSource* aMediaSource)
michael@0: {
michael@0:   MOZ_ASSERT(!mMediaSource && !mDecoderStateMachine);
michael@0:   mMediaSource = aMediaSource;
michael@0: }
michael@0: 
michael@0: void
michael@0: MediaSourceDecoder::DetachMediaSource()
michael@0: {
michael@0:   mMediaSource = nullptr;
michael@0: }
michael@0: 
michael@0: already_AddRefed<SubBufferDecoder>
michael@0: MediaSourceDecoder::CreateSubDecoder(const nsACString& aType)
michael@0: {
michael@0:   if (!mDecoderStateMachine) {
michael@0:     return nullptr;
michael@0:   }
michael@0:   return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->CreateSubDecoder(aType, this);
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaSourceDecoder::EnqueueDecoderInitialization()
michael@0: {
michael@0:   if (!mDecoderStateMachine) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0:   return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->EnqueueDecoderInitialization();
michael@0: }
michael@0: 
michael@0: class ReleaseDecodersTask : public nsRunnable {
michael@0: public:
michael@0:   ReleaseDecodersTask(nsTArray<nsRefPtr<SubBufferDecoder>>& aDecoders)
michael@0:   {
michael@0:     mDecoders.SwapElements(aDecoders);
michael@0:   }
michael@0: 
michael@0:   NS_IMETHOD Run() MOZ_OVERRIDE MOZ_FINAL {
michael@0:     mDecoders.Clear();
michael@0:     return NS_OK;
michael@0:   }
michael@0: 
michael@0: private:
michael@0:   nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
michael@0: };
michael@0: 
michael@0: void
michael@0: MediaSourceReader::CallDecoderInitialization()
michael@0: {
michael@0:   ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
michael@0:   for (uint32_t i = 0; i < mPendingDecoders.Length(); ++i) {
michael@0:     nsRefPtr<SubBufferDecoder> decoder = mPendingDecoders[i];
michael@0:     MediaDecoderReader* reader = decoder->GetReader();
michael@0:     MSE_DEBUG("%p: Initializating subdecoder %p reader %p", this, decoder.get(), reader);
michael@0: 
michael@0:     reader->SetActive();
michael@0:     MediaInfo mi;
michael@0:     nsAutoPtr<MetadataTags> tags; // TODO: Handle metadata.
michael@0:     nsresult rv;
michael@0:     {
michael@0:       ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
michael@0:       rv = reader->ReadMetadata(&mi, getter_Transfers(tags));
michael@0:     }
michael@0:     reader->SetIdle();
michael@0:     if (NS_FAILED(rv)) {
michael@0:       // XXX: Need to signal error back to owning SourceBuffer.
michael@0:       MSE_DEBUG("%p: Reader %p failed to initialize, rv=%x", this, reader, rv);
michael@0:       continue;
michael@0:     }
michael@0: 
michael@0:     bool active = false;
michael@0:     if (mi.HasVideo() || mi.HasAudio()) {
michael@0:       MSE_DEBUG("%p: Reader %p has video=%d audio=%d", this, reader, mi.HasVideo(), mi.HasAudio());
michael@0:       active = true;
michael@0:     }
michael@0: 
michael@0:     if (active) {
michael@0:       mDecoders.AppendElement(decoder);
michael@0:     } else {
michael@0:       MSE_DEBUG("%p: Reader %p not activated", this, reader);
michael@0:     }
michael@0:   }
michael@0:   NS_DispatchToMainThread(new ReleaseDecodersTask(mPendingDecoders));
michael@0:   MOZ_ASSERT(mPendingDecoders.IsEmpty());
michael@0:   mDecoder->NotifyWaitingForResourcesStatusChanged();
michael@0:   mon.NotifyAll();
michael@0: }
michael@0: 
michael@0: void
michael@0: MediaSourceReader::WaitForPendingDecoders()
michael@0: {
michael@0:   ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
michael@0:   while (!mPendingDecoders.IsEmpty()) {
michael@0:     mon.Wait();
michael@0:   }
michael@0: }
michael@0: 
michael@0: already_AddRefed<SubBufferDecoder>
michael@0: MediaSourceReader::CreateSubDecoder(const nsACString& aType, MediaSourceDecoder* aParentDecoder)
michael@0: {
michael@0:   // XXX: Why/when is mDecoder null here, since it should be equal to aParentDecoder?!
michael@0:   nsRefPtr<SubBufferDecoder> decoder =
michael@0:     new SubBufferDecoder(new SourceBufferResource(nullptr, aType), aParentDecoder);
michael@0:   nsAutoPtr<MediaDecoderReader> reader(DecoderTraits::CreateReader(aType, decoder));
michael@0:   if (!reader) {
michael@0:     return nullptr;
michael@0:   }
michael@0:   reader->Init(nullptr);
michael@0:   ReentrantMonitorAutoEnter mon(aParentDecoder->GetReentrantMonitor());
michael@0:   MSE_DEBUG("Registered subdecoder %p subreader %p", decoder.get(), reader.get());
michael@0:   decoder->SetReader(reader.forget());
michael@0:   mPendingDecoders.AppendElement(decoder);
michael@0:   if (NS_FAILED(static_cast<MediaSourceDecoder*>(mDecoder)->EnqueueDecoderInitialization())) {
michael@0:     MSE_DEBUG("%p: Failed to enqueue decoder initialization task", this);
michael@0:     return nullptr;
michael@0:   }
michael@0:   mDecoder->NotifyWaitingForResourcesStatusChanged();
michael@0:   return decoder.forget();
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaSourceReader::Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
michael@0:                         int64_t aCurrentTime)
michael@0: {
michael@0:   ResetDecode();
michael@0: 
michael@0:   dom::SourceBufferList* sbl = mMediaSource->ActiveSourceBuffers();
michael@0:   if (sbl->AllContainsTime (aTime / USECS_PER_S)) {
michael@0:     if (GetAudioReader()) {
michael@0:       nsresult rv = GetAudioReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
michael@0:       if (NS_FAILED(rv)) {
michael@0:         return rv;
michael@0:       }
michael@0:     }
michael@0:     if (GetVideoReader()) {
michael@0:       nsresult rv = GetVideoReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
michael@0:       if (NS_FAILED(rv)) {
michael@0:         return rv;
michael@0:       }
michael@0:     }
michael@0:   }
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaSourceReader::GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime)
michael@0: {
michael@0:   for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
michael@0:     nsRefPtr<dom::TimeRanges> r = new dom::TimeRanges();
michael@0:     mDecoders[i]->GetBuffered(r);
michael@0:     aBuffered->Add(r->GetStartTime(), r->GetEndTime());
michael@0:   }
michael@0:   aBuffered->Normalize();
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags)
michael@0: {
michael@0:   mDecoder->SetMediaSeekable(true);
michael@0:   mDecoder->SetTransportSeekable(false);
michael@0: 
michael@0:   MSE_DEBUG("%p: MSR::ReadMetadata pending=%u", this, mPendingDecoders.Length());
michael@0: 
michael@0:   WaitForPendingDecoders();
michael@0: 
michael@0:   MSE_DEBUG("%p: MSR::ReadMetadata decoders=%u", this, mDecoders.Length());
michael@0: 
michael@0:   // XXX: Make subdecoder setup async, so that use cases like bug 989888 can
michael@0:   // work.  This will require teaching the state machine about dynamic track
michael@0:   // changes (and multiple tracks).
michael@0:   // Shorter term, make this block until we've got at least one video track
michael@0:   // and lie about having an audio track, then resample/remix as necessary
michael@0:   // to match any audio track added later to fit the format we lied about
michael@0:   // now.  For now we just configure what we've got and cross our fingers.
michael@0:   int64_t maxDuration = -1;
michael@0:   for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
michael@0:     MediaDecoderReader* reader = mDecoders[i]->GetReader();
michael@0: 
michael@0:     reader->SetActive(); // XXX check where this should be called
michael@0: 
michael@0:     MediaInfo mi = reader->GetMediaInfo();
michael@0: 
michael@0:     if (mi.HasVideo() && !mInfo.HasVideo()) {
michael@0:       MOZ_ASSERT(mActiveVideoDecoder == -1);
michael@0:       mActiveVideoDecoder = i;
michael@0:       mInfo.mVideo = mi.mVideo;
michael@0:       maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
michael@0:       MSE_DEBUG("%p: MSR::ReadMetadata video decoder=%u maxDuration=%lld", this, i, maxDuration);
michael@0:     }
michael@0:     if (mi.HasAudio() && !mInfo.HasAudio()) {
michael@0:       MOZ_ASSERT(mActiveAudioDecoder == -1);
michael@0:       mActiveAudioDecoder = i;
michael@0:       mInfo.mAudio = mi.mAudio;
michael@0:       maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
michael@0:       MSE_DEBUG("%p: MSR::ReadMetadata audio decoder=%u maxDuration=%lld", this, i, maxDuration);
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   if (maxDuration != -1) {
michael@0:     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
michael@0:     mDecoder->SetMediaDuration(maxDuration);
michael@0:     ErrorResult dummy;
michael@0:     mMediaSource->SetDuration(maxDuration, dummy);
michael@0:   }
michael@0: 
michael@0:   *aInfo = mInfo;
michael@0:   *aTags = nullptr; // TODO: Handle metadata.
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: } // namespace mozilla