michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ michael@0: /* vim:set ts=2 sw=2 sts=2 et cindent: */ michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: #include "MP4Reader.h" michael@0: #include "MediaResource.h" michael@0: #include "mp4_demuxer/mp4_demuxer.h" michael@0: #include "mp4_demuxer/Streams.h" michael@0: #include "nsSize.h" michael@0: #include "VideoUtils.h" michael@0: #include "mozilla/dom/HTMLMediaElement.h" michael@0: #include "ImageContainer.h" michael@0: #include "Layers.h" michael@0: #include "SharedThreadPool.h" michael@0: #include "mozilla/Preferences.h" michael@0: michael@0: using mozilla::layers::Image; michael@0: using mozilla::layers::LayerManager; michael@0: using mozilla::layers::LayersBackend; michael@0: michael@0: #ifdef PR_LOGGING michael@0: PRLogModuleInfo* GetDemuxerLog() { michael@0: static PRLogModuleInfo* log = nullptr; michael@0: if (!log) { michael@0: log = PR_NewLogModule("MP4Demuxer"); michael@0: } michael@0: return log; michael@0: } michael@0: #define LOG(...) PR_LOG(GetDemuxerLog(), PR_LOG_DEBUG, (__VA_ARGS__)) michael@0: #else michael@0: #define LOG(...) michael@0: #endif michael@0: michael@0: using namespace mp4_demuxer; michael@0: michael@0: namespace mozilla { michael@0: michael@0: // Uncomment to enable verbose per-sample logging. michael@0: //#define LOG_SAMPLE_DECODE 1 michael@0: michael@0: class MP4Stream : public mp4_demuxer::Stream { michael@0: public: michael@0: michael@0: MP4Stream(MediaResource* aResource) michael@0: : mResource(aResource) michael@0: { michael@0: MOZ_COUNT_CTOR(MP4Stream); michael@0: MOZ_ASSERT(aResource); michael@0: } michael@0: virtual ~MP4Stream() { michael@0: MOZ_COUNT_DTOR(MP4Stream); michael@0: } michael@0: michael@0: virtual bool ReadAt(int64_t aOffset, michael@0: uint8_t* aBuffer, michael@0: uint32_t aCount, michael@0: uint32_t* aBytesRead) MOZ_OVERRIDE { michael@0: uint32_t sum = 0; michael@0: do { michael@0: uint32_t offset = aOffset + sum; michael@0: char* buffer = reinterpret_cast(aBuffer + sum); michael@0: uint32_t toRead = aCount - sum; michael@0: uint32_t bytesRead = 0; michael@0: nsresult rv = mResource->ReadAt(offset, buffer, toRead, &bytesRead); michael@0: if (NS_FAILED(rv)) { michael@0: return false; michael@0: } michael@0: sum += bytesRead; michael@0: } while (sum < aCount); michael@0: *aBytesRead = sum; michael@0: return true; michael@0: } michael@0: michael@0: virtual int64_t Length() const MOZ_OVERRIDE { michael@0: return mResource->GetLength(); michael@0: } michael@0: michael@0: private: michael@0: RefPtr mResource; michael@0: }; michael@0: michael@0: MP4Reader::MP4Reader(AbstractMediaDecoder* aDecoder) michael@0: : MediaDecoderReader(aDecoder) michael@0: , mAudio("MP4 audio decoder data", Preferences::GetUint("media.mp4-audio-decode-ahead", 2)) michael@0: , mVideo("MP4 video decoder data", Preferences::GetUint("media.mp4-video-decode-ahead", 2)) michael@0: , mLastReportedNumDecodedFrames(0) michael@0: , mLayersBackendType(layers::LayersBackend::LAYERS_NONE) michael@0: { michael@0: MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); michael@0: MOZ_COUNT_CTOR(MP4Reader); michael@0: } michael@0: michael@0: MP4Reader::~MP4Reader() michael@0: { michael@0: MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); michael@0: MOZ_COUNT_DTOR(MP4Reader); michael@0: Shutdown(); michael@0: } michael@0: michael@0: void michael@0: MP4Reader::Shutdown() michael@0: { michael@0: if (mAudio.mDecoder) { michael@0: Flush(kAudio); michael@0: mAudio.mDecoder->Shutdown(); michael@0: mAudio.mDecoder = nullptr; michael@0: } michael@0: if (mAudio.mTaskQueue) { michael@0: mAudio.mTaskQueue->Shutdown(); michael@0: mAudio.mTaskQueue = nullptr; michael@0: } michael@0: if (mVideo.mDecoder) { michael@0: Flush(kVideo); michael@0: mVideo.mDecoder->Shutdown(); michael@0: mVideo.mDecoder = nullptr; michael@0: } michael@0: if (mVideo.mTaskQueue) { michael@0: mVideo.mTaskQueue->Shutdown(); michael@0: mVideo.mTaskQueue = nullptr; michael@0: } michael@0: } michael@0: michael@0: void michael@0: MP4Reader::InitLayersBackendType() michael@0: { michael@0: if (!IsVideoContentType(mDecoder->GetResource()->GetContentType())) { michael@0: // Not playing video, we don't care about the layers backend type. michael@0: return; michael@0: } michael@0: // Extract the layer manager backend type so that platform decoders michael@0: // can determine whether it's worthwhile using hardware accelerated michael@0: // video decoding. michael@0: MediaDecoderOwner* owner = mDecoder->GetOwner(); michael@0: if (!owner) { michael@0: NS_WARNING("MP4Reader without a decoder owner, can't get HWAccel"); michael@0: return; michael@0: } michael@0: michael@0: dom::HTMLMediaElement* element = owner->GetMediaElement(); michael@0: NS_ENSURE_TRUE_VOID(element); michael@0: michael@0: nsRefPtr layerManager = michael@0: nsContentUtils::LayerManagerForDocument(element->OwnerDoc()); michael@0: NS_ENSURE_TRUE_VOID(layerManager); michael@0: michael@0: mLayersBackendType = layerManager->GetCompositorBackendType(); michael@0: } michael@0: michael@0: nsresult michael@0: MP4Reader::Init(MediaDecoderReader* aCloneDonor) michael@0: { michael@0: MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); michael@0: PlatformDecoderModule::Init(); michael@0: mMP4Stream = new MP4Stream(mDecoder->GetResource()); michael@0: mDemuxer = new MP4Demuxer(mMP4Stream); michael@0: michael@0: InitLayersBackendType(); michael@0: michael@0: mAudio.mTaskQueue = new MediaTaskQueue( michael@0: SharedThreadPool::Get(NS_LITERAL_CSTRING("MP4 Audio Decode"))); michael@0: NS_ENSURE_TRUE(mAudio.mTaskQueue, NS_ERROR_FAILURE); michael@0: michael@0: mVideo.mTaskQueue = new MediaTaskQueue( michael@0: SharedThreadPool::Get(NS_LITERAL_CSTRING("MP4 Video Decode"))); michael@0: NS_ENSURE_TRUE(mVideo.mTaskQueue, NS_ERROR_FAILURE); michael@0: michael@0: return NS_OK; michael@0: } michael@0: michael@0: nsresult michael@0: MP4Reader::ReadMetadata(MediaInfo* aInfo, michael@0: MetadataTags** aTags) michael@0: { michael@0: bool ok = mDemuxer->Init(); michael@0: NS_ENSURE_TRUE(ok, NS_ERROR_FAILURE); michael@0: michael@0: const AudioDecoderConfig& audio = mDemuxer->AudioConfig(); michael@0: mInfo.mAudio.mHasAudio = mAudio.mActive = mDemuxer->HasAudio() && michael@0: audio.IsValidConfig(); michael@0: // If we have audio, we *only* allow AAC to be decoded. michael@0: if (HasAudio() && audio.codec() != kCodecAAC) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: michael@0: const VideoDecoderConfig& video = mDemuxer->VideoConfig(); michael@0: mInfo.mVideo.mHasVideo = mVideo.mActive = mDemuxer->HasVideo() && michael@0: video.IsValidConfig(); michael@0: // If we have video, we *only* allow H.264 to be decoded. michael@0: if (HasVideo() && video.codec() != kCodecH264) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: michael@0: mPlatform = PlatformDecoderModule::Create(); michael@0: NS_ENSURE_TRUE(mPlatform, NS_ERROR_FAILURE); michael@0: michael@0: if (HasAudio()) { michael@0: mInfo.mAudio.mRate = audio.samples_per_second(); michael@0: mInfo.mAudio.mChannels = ChannelLayoutToChannelCount(audio.channel_layout()); michael@0: mAudio.mCallback = new DecoderCallback(this, kAudio); michael@0: mAudio.mDecoder = mPlatform->CreateAACDecoder(audio, michael@0: mAudio.mTaskQueue, michael@0: mAudio.mCallback); michael@0: NS_ENSURE_TRUE(mAudio.mDecoder != nullptr, NS_ERROR_FAILURE); michael@0: nsresult rv = mAudio.mDecoder->Init(); michael@0: NS_ENSURE_SUCCESS(rv, rv); michael@0: } michael@0: michael@0: if (HasVideo()) { michael@0: IntSize sz = video.natural_size(); michael@0: mInfo.mVideo.mDisplay = nsIntSize(sz.width(), sz.height()); michael@0: mVideo.mCallback = new DecoderCallback(this, kVideo); michael@0: mVideo.mDecoder = mPlatform->CreateH264Decoder(video, michael@0: mLayersBackendType, michael@0: mDecoder->GetImageContainer(), michael@0: mVideo.mTaskQueue, michael@0: mVideo.mCallback); michael@0: NS_ENSURE_TRUE(mVideo.mDecoder != nullptr, NS_ERROR_FAILURE); michael@0: nsresult rv = mVideo.mDecoder->Init(); michael@0: NS_ENSURE_SUCCESS(rv, rv); michael@0: } michael@0: michael@0: // Get the duration, and report it to the decoder if we have it. michael@0: Microseconds duration = mDemuxer->Duration(); michael@0: if (duration != -1) { michael@0: ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); michael@0: mDecoder->SetMediaDuration(duration); michael@0: } michael@0: // We can seek if we get a duration *and* the reader reports that it's michael@0: // seekable. michael@0: if (!mDemuxer->CanSeek()) { michael@0: mDecoder->SetMediaSeekable(false); michael@0: } michael@0: michael@0: *aInfo = mInfo; michael@0: *aTags = nullptr; michael@0: michael@0: return NS_OK; michael@0: } michael@0: michael@0: bool michael@0: MP4Reader::HasAudio() michael@0: { michael@0: return mAudio.mActive; michael@0: } michael@0: michael@0: bool michael@0: MP4Reader::HasVideo() michael@0: { michael@0: return mVideo.mActive; michael@0: } michael@0: michael@0: MP4Reader::DecoderData& michael@0: MP4Reader::GetDecoderData(TrackType aTrack) michael@0: { michael@0: MOZ_ASSERT(aTrack == kAudio || aTrack == kVideo); michael@0: return (aTrack == kAudio) ? mAudio : mVideo; michael@0: } michael@0: michael@0: MP4SampleQueue& michael@0: MP4Reader::SampleQueue(TrackType aTrack) michael@0: { michael@0: return GetDecoderData(aTrack).mDemuxedSamples; michael@0: } michael@0: michael@0: MediaDataDecoder* michael@0: MP4Reader::Decoder(mp4_demuxer::TrackType aTrack) michael@0: { michael@0: return GetDecoderData(aTrack).mDecoder; michael@0: } michael@0: michael@0: MP4Sample* michael@0: MP4Reader::PopSample(TrackType aTrack) michael@0: { michael@0: // Unfortunately the demuxer outputs in the order samples appear in the michael@0: // media, not on a per stream basis. We cache the samples we get from michael@0: // streams other than the one we want. michael@0: MP4SampleQueue& sampleQueue = SampleQueue(aTrack); michael@0: while (sampleQueue.empty()) { michael@0: nsAutoPtr sample; michael@0: bool eos = false; michael@0: bool ok = mDemuxer->Demux(&sample, &eos); michael@0: if (!ok || eos) { michael@0: MOZ_ASSERT(!sample); michael@0: return nullptr; michael@0: } michael@0: MOZ_ASSERT(sample); michael@0: MP4Sample* s = sample.forget(); michael@0: SampleQueue(s->type).push_back(s); michael@0: } michael@0: MOZ_ASSERT(!sampleQueue.empty()); michael@0: MP4Sample* sample = sampleQueue.front(); michael@0: sampleQueue.pop_front(); michael@0: return sample; michael@0: } michael@0: michael@0: // How async decoding works: michael@0: // michael@0: // When MP4Reader::Decode() is called: michael@0: // * Lock the DecoderData. We assume the state machine wants michael@0: // output from the decoder (in future, we'll assume decoder wants input michael@0: // when the output MediaQueue isn't "full"). michael@0: // * Cache the value of mNumSamplesOutput, as prevFramesOutput. michael@0: // * While we've not output data (mNumSamplesOutput != prevNumFramesOutput) michael@0: // and while we still require input, we demux and input data in the reader. michael@0: // We assume we require input if michael@0: // ((mNumSamplesInput - mNumSamplesOutput) < sDecodeAheadMargin) or michael@0: // mInputExhausted is true. Before we send input, we reset mInputExhausted michael@0: // and increment mNumFrameInput, and drop the lock on DecoderData. michael@0: // * Once we no longer require input, we wait on the DecoderData michael@0: // lock for output, or for the input exhausted callback. If we receive the michael@0: // input exhausted callback, we go back and input more data. michael@0: // * When our output callback is called, we take the DecoderData lock and michael@0: // increment mNumSamplesOutput. We notify the DecoderData lock. This will michael@0: // awaken the Decode thread, and unblock it, and it will return. michael@0: bool michael@0: MP4Reader::Decode(TrackType aTrack) michael@0: { michael@0: DecoderData& data = GetDecoderData(aTrack); michael@0: MOZ_ASSERT(data.mDecoder); michael@0: michael@0: data.mMonitor.Lock(); michael@0: uint64_t prevNumFramesOutput = data.mNumSamplesOutput; michael@0: while (prevNumFramesOutput == data.mNumSamplesOutput) { michael@0: data.mMonitor.AssertCurrentThreadOwns(); michael@0: if (data.mError) { michael@0: // Decode error! michael@0: data.mMonitor.Unlock(); michael@0: return false; michael@0: } michael@0: // Send input to the decoder, if we need to. We assume the decoder michael@0: // needs input if it's told us it's out of input, or we're beneath the michael@0: // "low water mark" for the amount of input we've sent it vs the amount michael@0: // out output we've received. We always try to keep the decoder busy if michael@0: // possible, so we try to maintain at least a few input samples ahead, michael@0: // if we need output. michael@0: while (prevNumFramesOutput == data.mNumSamplesOutput && michael@0: (data.mInputExhausted || michael@0: (data.mNumSamplesInput - data.mNumSamplesOutput) < data.mDecodeAhead)) { michael@0: data.mMonitor.AssertCurrentThreadOwns(); michael@0: data.mMonitor.Unlock(); michael@0: nsAutoPtr compressed(PopSample(aTrack)); michael@0: if (!compressed) { michael@0: // EOS, or error. Let the state machine know there are no more michael@0: // frames coming. michael@0: return false; michael@0: } michael@0: data.mMonitor.Lock(); michael@0: data.mInputExhausted = false; michael@0: data.mNumSamplesInput++; michael@0: data.mMonitor.Unlock(); michael@0: if (NS_FAILED(data.mDecoder->Input(compressed))) { michael@0: return false; michael@0: } michael@0: // If Input() failed, we let the auto pointer delete |compressed|. michael@0: // Otherwise, we assume the decoder will delete it when it's finished michael@0: // with it. michael@0: compressed.forget(); michael@0: data.mMonitor.Lock(); michael@0: } michael@0: data.mMonitor.AssertCurrentThreadOwns(); michael@0: while (!data.mError && michael@0: prevNumFramesOutput == data.mNumSamplesOutput && michael@0: !data.mInputExhausted ) { michael@0: data.mMonitor.Wait(); michael@0: } michael@0: } michael@0: data.mMonitor.AssertCurrentThreadOwns(); michael@0: data.mMonitor.Unlock(); michael@0: return true; michael@0: } michael@0: michael@0: #ifdef LOG_SAMPLE_DECODE michael@0: static const char* michael@0: TrackTypeToStr(TrackType aTrack) michael@0: { michael@0: MOZ_ASSERT(aTrack == kAudio || aTrack == kVideo); michael@0: switch (aTrack) { michael@0: case kAudio: return "Audio"; michael@0: case kVideo: return "Video"; michael@0: default: return "Unknown"; michael@0: } michael@0: } michael@0: #endif michael@0: michael@0: void michael@0: MP4Reader::Output(mp4_demuxer::TrackType aTrack, MediaData* aSample) michael@0: { michael@0: #ifdef LOG_SAMPLE_DECODE michael@0: LOG("Decoded %s sample time=%lld dur=%lld", michael@0: TrackTypeToStr(aTrack), aSample->mTime, aSample->mDuration); michael@0: #endif michael@0: michael@0: DecoderData& data = GetDecoderData(aTrack); michael@0: // Don't accept output while we're flushing. michael@0: MonitorAutoLock mon(data.mMonitor); michael@0: if (data.mIsFlushing) { michael@0: mon.NotifyAll(); michael@0: return; michael@0: } michael@0: michael@0: switch (aTrack) { michael@0: case kAudio: { michael@0: MOZ_ASSERT(aSample->mType == MediaData::AUDIO_SAMPLES); michael@0: AudioQueue().Push(static_cast(aSample)); michael@0: break; michael@0: } michael@0: case kVideo: { michael@0: MOZ_ASSERT(aSample->mType == MediaData::VIDEO_FRAME); michael@0: VideoQueue().Push(static_cast(aSample)); michael@0: break; michael@0: } michael@0: default: michael@0: break; michael@0: } michael@0: michael@0: data.mNumSamplesOutput++; michael@0: mon.NotifyAll(); michael@0: } michael@0: michael@0: void michael@0: MP4Reader::InputExhausted(mp4_demuxer::TrackType aTrack) michael@0: { michael@0: DecoderData& data = GetDecoderData(aTrack); michael@0: MonitorAutoLock mon(data.mMonitor); michael@0: data.mInputExhausted = true; michael@0: mon.NotifyAll(); michael@0: } michael@0: michael@0: void michael@0: MP4Reader::Error(mp4_demuxer::TrackType aTrack) michael@0: { michael@0: DecoderData& data = GetDecoderData(aTrack); michael@0: MonitorAutoLock mon(data.mMonitor); michael@0: data.mError = true; michael@0: mon.NotifyAll(); michael@0: } michael@0: michael@0: bool michael@0: MP4Reader::DecodeAudioData() michael@0: { michael@0: MOZ_ASSERT(HasAudio() && mPlatform && mAudio.mDecoder); michael@0: return Decode(kAudio); michael@0: } michael@0: michael@0: void michael@0: MP4Reader::Flush(mp4_demuxer::TrackType aTrack) michael@0: { michael@0: DecoderData& data = GetDecoderData(aTrack); michael@0: if (!data.mDecoder) { michael@0: return; michael@0: } michael@0: // Purge the current decoder's state. michael@0: // Set a flag so that we ignore all output while we call michael@0: // MediaDataDecoder::Flush(). michael@0: { michael@0: data.mIsFlushing = true; michael@0: MonitorAutoLock mon(data.mMonitor); michael@0: } michael@0: data.mDecoder->Flush(); michael@0: { michael@0: data.mIsFlushing = false; michael@0: MonitorAutoLock mon(data.mMonitor); michael@0: } michael@0: } michael@0: michael@0: bool michael@0: MP4Reader::SkipVideoDemuxToNextKeyFrame(int64_t aTimeThreshold, uint32_t& parsed) michael@0: { michael@0: MOZ_ASSERT(mVideo.mDecoder); michael@0: michael@0: Flush(kVideo); michael@0: michael@0: // Loop until we reach the next keyframe after the threshold. michael@0: while (true) { michael@0: nsAutoPtr compressed(PopSample(kVideo)); michael@0: if (!compressed) { michael@0: // EOS, or error. Let the state machine know. michael@0: return false; michael@0: } michael@0: parsed++; michael@0: if (!compressed->is_sync_point || michael@0: compressed->composition_timestamp < aTimeThreshold) { michael@0: continue; michael@0: } michael@0: mVideo.mDemuxedSamples.push_front(compressed.forget()); michael@0: break; michael@0: } michael@0: michael@0: return true; michael@0: } michael@0: michael@0: bool michael@0: MP4Reader::DecodeVideoFrame(bool &aKeyframeSkip, michael@0: int64_t aTimeThreshold) michael@0: { michael@0: // Record number of frames decoded and parsed. Automatically update the michael@0: // stats counters using the AutoNotifyDecoded stack-based class. michael@0: uint32_t parsed = 0, decoded = 0; michael@0: AbstractMediaDecoder::AutoNotifyDecoded autoNotify(mDecoder, parsed, decoded); michael@0: michael@0: MOZ_ASSERT(HasVideo() && mPlatform && mVideo.mDecoder); michael@0: michael@0: if (aKeyframeSkip) { michael@0: bool ok = SkipVideoDemuxToNextKeyFrame(aTimeThreshold, parsed); michael@0: if (!ok) { michael@0: NS_WARNING("Failed to skip demux up to next keyframe"); michael@0: return false; michael@0: } michael@0: aKeyframeSkip = false; michael@0: nsresult rv = mVideo.mDecoder->Flush(); michael@0: NS_ENSURE_SUCCESS(rv, false); michael@0: } michael@0: michael@0: bool rv = Decode(kVideo); michael@0: { michael@0: // Report the number of "decoded" frames as the difference in the michael@0: // mNumSamplesOutput field since the last time we were called. michael@0: MonitorAutoLock mon(mVideo.mMonitor); michael@0: uint64_t delta = mVideo.mNumSamplesOutput - mLastReportedNumDecodedFrames; michael@0: decoded = static_cast(delta); michael@0: mLastReportedNumDecodedFrames = mVideo.mNumSamplesOutput; michael@0: } michael@0: return rv; michael@0: } michael@0: michael@0: nsresult michael@0: MP4Reader::Seek(int64_t aTime, michael@0: int64_t aStartTime, michael@0: int64_t aEndTime, michael@0: int64_t aCurrentTime) michael@0: { michael@0: if (!mDemuxer->CanSeek()) { michael@0: return NS_ERROR_FAILURE; michael@0: } michael@0: return NS_ERROR_NOT_IMPLEMENTED; michael@0: } michael@0: michael@0: } // namespace mozilla