michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* vim: set ts=8 sts=2 et sw=2 tw=80: */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "MediaDecoderReader.h"
michael@0: #include "AbstractMediaDecoder.h"
michael@0: #include "VideoUtils.h"
michael@0: #include "ImageContainer.h"
michael@0: 
michael@0: #include "mozilla/mozalloc.h"
michael@0: #include <stdint.h>
michael@0: #include <algorithm>
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: // Un-comment to enable logging of seek bisections.
michael@0: //#define SEEK_LOGGING
michael@0: 
michael@0: #ifdef PR_LOGGING
michael@0: extern PRLogModuleInfo* gMediaDecoderLog;
michael@0: #define DECODER_LOG(type, msg) PR_LOG(gMediaDecoderLog, type, msg)
michael@0: #ifdef SEEK_LOGGING
michael@0: #define SEEK_LOG(type, msg) PR_LOG(gMediaDecoderLog, type, msg)
michael@0: #else
michael@0: #define SEEK_LOG(type, msg)
michael@0: #endif
michael@0: #else
michael@0: #define DECODER_LOG(type, msg)
michael@0: #define SEEK_LOG(type, msg)
michael@0: #endif
michael@0: 
michael@0: class VideoQueueMemoryFunctor : public nsDequeFunctor {
michael@0: public:
michael@0:   VideoQueueMemoryFunctor() : mSize(0) {}
michael@0: 
michael@0:   MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf);
michael@0: 
michael@0:   virtual void* operator()(void* aObject) {
michael@0:     const VideoData* v = static_cast<const VideoData*>(aObject);
michael@0:     mSize += v->SizeOfIncludingThis(MallocSizeOf);
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   size_t mSize;
michael@0: };
michael@0: 
michael@0: 
michael@0: class AudioQueueMemoryFunctor : public nsDequeFunctor {
michael@0: public:
michael@0:   AudioQueueMemoryFunctor() : mSize(0) {}
michael@0: 
michael@0:   MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf);
michael@0: 
michael@0:   virtual void* operator()(void* aObject) {
michael@0:     const AudioData* audioData = static_cast<const AudioData*>(aObject);
michael@0:     mSize += audioData->SizeOfIncludingThis(MallocSizeOf);
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   size_t mSize;
michael@0: };
michael@0: 
michael@0: MediaDecoderReader::MediaDecoderReader(AbstractMediaDecoder* aDecoder)
michael@0:   : mAudioCompactor(mAudioQueue),
michael@0:     mDecoder(aDecoder),
michael@0:     mIgnoreAudioOutputFormat(false)
michael@0: {
michael@0:   MOZ_COUNT_CTOR(MediaDecoderReader);
michael@0: }
michael@0: 
michael@0: MediaDecoderReader::~MediaDecoderReader()
michael@0: {
michael@0:   ResetDecode();
michael@0:   MOZ_COUNT_DTOR(MediaDecoderReader);
michael@0: }
michael@0: 
michael@0: size_t MediaDecoderReader::SizeOfVideoQueueInBytes() const
michael@0: {
michael@0:   VideoQueueMemoryFunctor functor;
michael@0:   mVideoQueue.LockedForEach(functor);
michael@0:   return functor.mSize;
michael@0: }
michael@0: 
michael@0: size_t MediaDecoderReader::SizeOfAudioQueueInBytes() const
michael@0: {
michael@0:   AudioQueueMemoryFunctor functor;
michael@0:   mAudioQueue.LockedForEach(functor);
michael@0:   return functor.mSize;
michael@0: }
michael@0: 
michael@0: nsresult MediaDecoderReader::ResetDecode()
michael@0: {
michael@0:   nsresult res = NS_OK;
michael@0: 
michael@0:   VideoQueue().Reset();
michael@0:   AudioQueue().Reset();
michael@0: 
michael@0:   return res;
michael@0: }
michael@0: 
michael@0: VideoData* MediaDecoderReader::DecodeToFirstVideoData()
michael@0: {
michael@0:   bool eof = false;
michael@0:   while (!eof && VideoQueue().GetSize() == 0) {
michael@0:     {
michael@0:       ReentrantMonitorAutoEnter decoderMon(mDecoder->GetReentrantMonitor());
michael@0:       if (mDecoder->IsShutdown()) {
michael@0:         return nullptr;
michael@0:       }
michael@0:     }
michael@0:     bool keyframeSkip = false;
michael@0:     eof = !DecodeVideoFrame(keyframeSkip, 0);
michael@0:   }
michael@0:   if (eof) {
michael@0:     VideoQueue().Finish();
michael@0:   }
michael@0:   VideoData* d = nullptr;
michael@0:   return (d = VideoQueue().PeekFront()) ? d : nullptr;
michael@0: }
michael@0: 
michael@0: AudioData* MediaDecoderReader::DecodeToFirstAudioData()
michael@0: {
michael@0:   bool eof = false;
michael@0:   while (!eof && AudioQueue().GetSize() == 0) {
michael@0:     {
michael@0:       ReentrantMonitorAutoEnter decoderMon(mDecoder->GetReentrantMonitor());
michael@0:       if (mDecoder->IsShutdown()) {
michael@0:         return nullptr;
michael@0:       }
michael@0:     }
michael@0:     eof = !DecodeAudioData();
michael@0:   }
michael@0:   if (eof) {
michael@0:     AudioQueue().Finish();
michael@0:   }
michael@0:   AudioData* d = nullptr;
michael@0:   return (d = AudioQueue().PeekFront()) ? d : nullptr;
michael@0: }
michael@0: 
michael@0: VideoData* MediaDecoderReader::FindStartTime(int64_t& aOutStartTime)
michael@0: {
michael@0:   NS_ASSERTION(mDecoder->OnStateMachineThread() || mDecoder->OnDecodeThread(),
michael@0:                "Should be on state machine or decode thread.");
michael@0: 
michael@0:   // Extract the start times of the bitstreams in order to calculate
michael@0:   // the duration.
michael@0:   int64_t videoStartTime = INT64_MAX;
michael@0:   int64_t audioStartTime = INT64_MAX;
michael@0:   VideoData* videoData = nullptr;
michael@0: 
michael@0:   if (HasVideo()) {
michael@0:     videoData = DecodeToFirstVideoData();
michael@0:     if (videoData) {
michael@0:       videoStartTime = videoData->mTime;
michael@0:       DECODER_LOG(PR_LOG_DEBUG, ("MediaDecoderReader::FindStartTime() video=%lld", videoStartTime));
michael@0:     }
michael@0:   }
michael@0:   if (HasAudio()) {
michael@0:     AudioData* audioData = DecodeToFirstAudioData();
michael@0:     if (audioData) {
michael@0:       audioStartTime = audioData->mTime;
michael@0:       DECODER_LOG(PR_LOG_DEBUG, ("MediaDecoderReader::FindStartTime() audio=%lld", audioStartTime));
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   int64_t startTime = std::min(videoStartTime, audioStartTime);
michael@0:   if (startTime != INT64_MAX) {
michael@0:     aOutStartTime = startTime;
michael@0:   }
michael@0: 
michael@0:   return videoData;
michael@0: }
michael@0: 
michael@0: nsresult MediaDecoderReader::DecodeToTarget(int64_t aTarget)
michael@0: {
michael@0:   DECODER_LOG(PR_LOG_DEBUG, ("MediaDecoderReader::DecodeToTarget(%lld) Begin", aTarget));
michael@0: 
michael@0:   // Decode forward to the target frame. Start with video, if we have it.
michael@0:   if (HasVideo()) {
michael@0:     // Note: when decoding hits the end of stream we must keep the last frame
michael@0:     // in the video queue so that we'll have something to display after the
michael@0:     // seek completes. This makes our logic a bit messy.
michael@0:     bool eof = false;
michael@0:     nsAutoPtr<VideoData> video;
michael@0:     while (HasVideo() && !eof) {
michael@0:       while (VideoQueue().GetSize() == 0 && !eof) {
michael@0:         bool skip = false;
michael@0:         eof = !DecodeVideoFrame(skip, 0);
michael@0:         {
michael@0:           ReentrantMonitorAutoEnter decoderMon(mDecoder->GetReentrantMonitor());
michael@0:           if (mDecoder->IsShutdown()) {
michael@0:             return NS_ERROR_FAILURE;
michael@0:           }
michael@0:         }
michael@0:       }
michael@0:       if (eof) {
michael@0:         // Hit end of file, we want to display the last frame of the video.
michael@0:         if (video) {
michael@0:           DECODER_LOG(PR_LOG_DEBUG,
michael@0:             ("MediaDecoderReader::DecodeToTarget(%lld) repushing video frame [%lld, %lld] at EOF",
michael@0:             aTarget, video->mTime, video->GetEndTime()));
michael@0:           VideoQueue().PushFront(video.forget());
michael@0:         }
michael@0:         VideoQueue().Finish();
michael@0:         break;
michael@0:       }
michael@0:       video = VideoQueue().PeekFront();
michael@0:       // If the frame end time is less than the seek target, we won't want
michael@0:       // to display this frame after the seek, so discard it.
michael@0:       if (video && video->GetEndTime() <= aTarget) {
michael@0:         DECODER_LOG(PR_LOG_DEBUG,
michael@0:                     ("MediaDecoderReader::DecodeToTarget(%lld) pop video frame [%lld, %lld]",
michael@0:                      aTarget, video->mTime, video->GetEndTime()));
michael@0:         VideoQueue().PopFront();
michael@0:       } else {
michael@0:         // Found a frame after or encompasing the seek target.
michael@0:         if (aTarget >= video->mTime && video->GetEndTime() >= aTarget) {
michael@0:           // The seek target lies inside this frame's time slice. Adjust the frame's
michael@0:           // start time to match the seek target. We do this by replacing the
michael@0:           // first frame with a shallow copy which has the new timestamp.
michael@0:           VideoQueue().PopFront();
michael@0:           VideoData* temp = VideoData::ShallowCopyUpdateTimestamp(video, aTarget);
michael@0:           video = temp;
michael@0:           VideoQueue().PushFront(video);
michael@0:         }
michael@0:         DECODER_LOG(PR_LOG_DEBUG,
michael@0:                     ("MediaDecoderReader::DecodeToTarget(%lld) found target video frame [%lld,%lld]",
michael@0:                      aTarget, video->mTime, video->GetEndTime()));
michael@0: 
michael@0:         video.forget();
michael@0:         break;
michael@0:       }
michael@0:     }
michael@0:     {
michael@0:       ReentrantMonitorAutoEnter decoderMon(mDecoder->GetReentrantMonitor());
michael@0:       if (mDecoder->IsShutdown()) {
michael@0:         return NS_ERROR_FAILURE;
michael@0:       }
michael@0:     }
michael@0: #ifdef PR_LOGGING
michael@0:     const VideoData* front =  VideoQueue().PeekFront();
michael@0:     DECODER_LOG(PR_LOG_DEBUG, ("First video frame after decode is %lld",
michael@0:                 front ? front->mTime : -1));
michael@0: #endif
michael@0:   }
michael@0: 
michael@0:   if (HasAudio()) {
michael@0:     // Decode audio forward to the seek target.
michael@0:     bool eof = false;
michael@0:     while (HasAudio() && !eof) {
michael@0:       while (!eof && AudioQueue().GetSize() == 0) {
michael@0:         eof = !DecodeAudioData();
michael@0:         {
michael@0:           ReentrantMonitorAutoEnter decoderMon(mDecoder->GetReentrantMonitor());
michael@0:           if (mDecoder->IsShutdown()) {
michael@0:             return NS_ERROR_FAILURE;
michael@0:           }
michael@0:         }
michael@0:       }
michael@0:       const AudioData* audio = AudioQueue().PeekFront();
michael@0:       if (!audio || eof) {
michael@0:         AudioQueue().Finish();
michael@0:         break;
michael@0:       }
michael@0:       CheckedInt64 startFrame = UsecsToFrames(audio->mTime, mInfo.mAudio.mRate);
michael@0:       CheckedInt64 targetFrame = UsecsToFrames(aTarget, mInfo.mAudio.mRate);
michael@0:       if (!startFrame.isValid() || !targetFrame.isValid()) {
michael@0:         return NS_ERROR_FAILURE;
michael@0:       }
michael@0:       if (startFrame.value() + audio->mFrames <= targetFrame.value()) {
michael@0:         // Our seek target lies after the frames in this AudioData. Pop it
michael@0:         // off the queue, and keep decoding forwards.
michael@0:         delete AudioQueue().PopFront();
michael@0:         audio = nullptr;
michael@0:         continue;
michael@0:       }
michael@0:       if (startFrame.value() > targetFrame.value()) {
michael@0:         // The seek target doesn't lie in the audio block just after the last
michael@0:         // audio frames we've seen which were before the seek target. This
michael@0:         // could have been the first audio data we've seen after seek, i.e. the
michael@0:         // seek terminated after the seek target in the audio stream. Just
michael@0:         // abort the audio decode-to-target, the state machine will play
michael@0:         // silence to cover the gap. Typically this happens in poorly muxed
michael@0:         // files.
michael@0:         NS_WARNING("Audio not synced after seek, maybe a poorly muxed file?");
michael@0:         break;
michael@0:       }
michael@0: 
michael@0:       // The seek target lies somewhere in this AudioData's frames, strip off
michael@0:       // any frames which lie before the seek target, so we'll begin playback
michael@0:       // exactly at the seek target.
michael@0:       NS_ASSERTION(targetFrame.value() >= startFrame.value(),
michael@0:                    "Target must at or be after data start.");
michael@0:       NS_ASSERTION(targetFrame.value() < startFrame.value() + audio->mFrames,
michael@0:                    "Data must end after target.");
michael@0: 
michael@0:       int64_t framesToPrune = targetFrame.value() - startFrame.value();
michael@0:       if (framesToPrune > audio->mFrames) {
michael@0:         // We've messed up somehow. Don't try to trim frames, the |frames|
michael@0:         // variable below will overflow.
michael@0:         NS_WARNING("Can't prune more frames that we have!");
michael@0:         break;
michael@0:       }
michael@0:       uint32_t frames = audio->mFrames - static_cast<uint32_t>(framesToPrune);
michael@0:       uint32_t channels = audio->mChannels;
michael@0:       nsAutoArrayPtr<AudioDataValue> audioData(new AudioDataValue[frames * channels]);
michael@0:       memcpy(audioData.get(),
michael@0:              audio->mAudioData.get() + (framesToPrune * channels),
michael@0:              frames * channels * sizeof(AudioDataValue));
michael@0:       CheckedInt64 duration = FramesToUsecs(frames, mInfo.mAudio.mRate);
michael@0:       if (!duration.isValid()) {
michael@0:         return NS_ERROR_FAILURE;
michael@0:       }
michael@0:       nsAutoPtr<AudioData> data(new AudioData(audio->mOffset,
michael@0:                                               aTarget,
michael@0:                                               duration.value(),
michael@0:                                               frames,
michael@0:                                               audioData.forget(),
michael@0:                                               channels));
michael@0:       delete AudioQueue().PopFront();
michael@0:       AudioQueue().PushFront(data.forget());
michael@0:       break;
michael@0:     }
michael@0:   }
michael@0: 
michael@0: #ifdef PR_LOGGING
michael@0:   const VideoData* v = VideoQueue().PeekFront();
michael@0:   const AudioData* a = AudioQueue().PeekFront();
michael@0:   DECODER_LOG(PR_LOG_DEBUG,
michael@0:               ("MediaDecoderReader::DecodeToTarget(%lld) finished v=%lld a=%lld",
michael@0:               aTarget, v ? v->mTime : -1, a ? a->mTime : -1));
michael@0: #endif
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaDecoderReader::GetBuffered(mozilla::dom::TimeRanges* aBuffered,
michael@0:                                 int64_t aStartTime)
michael@0: {
michael@0:   MediaResource* stream = mDecoder->GetResource();
michael@0:   int64_t durationUs = 0;
michael@0:   {
michael@0:     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
michael@0:     durationUs = mDecoder->GetMediaDuration();
michael@0:   }
michael@0:   GetEstimatedBufferedTimeRanges(stream, durationUs, aBuffered);
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: } // namespace mozilla