1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/content/media/fmp4/MP4Reader.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,540 @@
1.4 +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
1.5 +/* vim:set ts=2 sw=2 sts=2 et cindent: */
1.6 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.7 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.9 +
1.10 +#include "MP4Reader.h"
1.11 +#include "MediaResource.h"
1.12 +#include "mp4_demuxer/mp4_demuxer.h"
1.13 +#include "mp4_demuxer/Streams.h"
1.14 +#include "nsSize.h"
1.15 +#include "VideoUtils.h"
1.16 +#include "mozilla/dom/HTMLMediaElement.h"
1.17 +#include "ImageContainer.h"
1.18 +#include "Layers.h"
1.19 +#include "SharedThreadPool.h"
1.20 +#include "mozilla/Preferences.h"
1.21 +
1.22 +using mozilla::layers::Image;
1.23 +using mozilla::layers::LayerManager;
1.24 +using mozilla::layers::LayersBackend;
1.25 +
1.26 +#ifdef PR_LOGGING
1.27 +PRLogModuleInfo* GetDemuxerLog() {
1.28 + static PRLogModuleInfo* log = nullptr;
1.29 + if (!log) {
1.30 + log = PR_NewLogModule("MP4Demuxer");
1.31 + }
1.32 + return log;
1.33 +}
1.34 +#define LOG(...) PR_LOG(GetDemuxerLog(), PR_LOG_DEBUG, (__VA_ARGS__))
1.35 +#else
1.36 +#define LOG(...)
1.37 +#endif
1.38 +
1.39 +using namespace mp4_demuxer;
1.40 +
1.41 +namespace mozilla {
1.42 +
1.43 +// Uncomment to enable verbose per-sample logging.
1.44 +//#define LOG_SAMPLE_DECODE 1
1.45 +
1.46 +class MP4Stream : public mp4_demuxer::Stream {
1.47 +public:
1.48 +
1.49 + MP4Stream(MediaResource* aResource)
1.50 + : mResource(aResource)
1.51 + {
1.52 + MOZ_COUNT_CTOR(MP4Stream);
1.53 + MOZ_ASSERT(aResource);
1.54 + }
1.55 + virtual ~MP4Stream() {
1.56 + MOZ_COUNT_DTOR(MP4Stream);
1.57 + }
1.58 +
1.59 + virtual bool ReadAt(int64_t aOffset,
1.60 + uint8_t* aBuffer,
1.61 + uint32_t aCount,
1.62 + uint32_t* aBytesRead) MOZ_OVERRIDE {
1.63 + uint32_t sum = 0;
1.64 + do {
1.65 + uint32_t offset = aOffset + sum;
1.66 + char* buffer = reinterpret_cast<char*>(aBuffer + sum);
1.67 + uint32_t toRead = aCount - sum;
1.68 + uint32_t bytesRead = 0;
1.69 + nsresult rv = mResource->ReadAt(offset, buffer, toRead, &bytesRead);
1.70 + if (NS_FAILED(rv)) {
1.71 + return false;
1.72 + }
1.73 + sum += bytesRead;
1.74 + } while (sum < aCount);
1.75 + *aBytesRead = sum;
1.76 + return true;
1.77 + }
1.78 +
1.79 + virtual int64_t Length() const MOZ_OVERRIDE {
1.80 + return mResource->GetLength();
1.81 + }
1.82 +
1.83 +private:
1.84 + RefPtr<MediaResource> mResource;
1.85 +};
1.86 +
1.87 +MP4Reader::MP4Reader(AbstractMediaDecoder* aDecoder)
1.88 + : MediaDecoderReader(aDecoder)
1.89 + , mAudio("MP4 audio decoder data", Preferences::GetUint("media.mp4-audio-decode-ahead", 2))
1.90 + , mVideo("MP4 video decoder data", Preferences::GetUint("media.mp4-video-decode-ahead", 2))
1.91 + , mLastReportedNumDecodedFrames(0)
1.92 + , mLayersBackendType(layers::LayersBackend::LAYERS_NONE)
1.93 +{
1.94 + MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
1.95 + MOZ_COUNT_CTOR(MP4Reader);
1.96 +}
1.97 +
1.98 +MP4Reader::~MP4Reader()
1.99 +{
1.100 + MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
1.101 + MOZ_COUNT_DTOR(MP4Reader);
1.102 + Shutdown();
1.103 +}
1.104 +
1.105 +void
1.106 +MP4Reader::Shutdown()
1.107 +{
1.108 + if (mAudio.mDecoder) {
1.109 + Flush(kAudio);
1.110 + mAudio.mDecoder->Shutdown();
1.111 + mAudio.mDecoder = nullptr;
1.112 + }
1.113 + if (mAudio.mTaskQueue) {
1.114 + mAudio.mTaskQueue->Shutdown();
1.115 + mAudio.mTaskQueue = nullptr;
1.116 + }
1.117 + if (mVideo.mDecoder) {
1.118 + Flush(kVideo);
1.119 + mVideo.mDecoder->Shutdown();
1.120 + mVideo.mDecoder = nullptr;
1.121 + }
1.122 + if (mVideo.mTaskQueue) {
1.123 + mVideo.mTaskQueue->Shutdown();
1.124 + mVideo.mTaskQueue = nullptr;
1.125 + }
1.126 +}
1.127 +
1.128 +void
1.129 +MP4Reader::InitLayersBackendType()
1.130 +{
1.131 + if (!IsVideoContentType(mDecoder->GetResource()->GetContentType())) {
1.132 + // Not playing video, we don't care about the layers backend type.
1.133 + return;
1.134 + }
1.135 + // Extract the layer manager backend type so that platform decoders
1.136 + // can determine whether it's worthwhile using hardware accelerated
1.137 + // video decoding.
1.138 + MediaDecoderOwner* owner = mDecoder->GetOwner();
1.139 + if (!owner) {
1.140 + NS_WARNING("MP4Reader without a decoder owner, can't get HWAccel");
1.141 + return;
1.142 + }
1.143 +
1.144 + dom::HTMLMediaElement* element = owner->GetMediaElement();
1.145 + NS_ENSURE_TRUE_VOID(element);
1.146 +
1.147 + nsRefPtr<LayerManager> layerManager =
1.148 + nsContentUtils::LayerManagerForDocument(element->OwnerDoc());
1.149 + NS_ENSURE_TRUE_VOID(layerManager);
1.150 +
1.151 + mLayersBackendType = layerManager->GetCompositorBackendType();
1.152 +}
1.153 +
1.154 +nsresult
1.155 +MP4Reader::Init(MediaDecoderReader* aCloneDonor)
1.156 +{
1.157 + MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
1.158 + PlatformDecoderModule::Init();
1.159 + mMP4Stream = new MP4Stream(mDecoder->GetResource());
1.160 + mDemuxer = new MP4Demuxer(mMP4Stream);
1.161 +
1.162 + InitLayersBackendType();
1.163 +
1.164 + mAudio.mTaskQueue = new MediaTaskQueue(
1.165 + SharedThreadPool::Get(NS_LITERAL_CSTRING("MP4 Audio Decode")));
1.166 + NS_ENSURE_TRUE(mAudio.mTaskQueue, NS_ERROR_FAILURE);
1.167 +
1.168 + mVideo.mTaskQueue = new MediaTaskQueue(
1.169 + SharedThreadPool::Get(NS_LITERAL_CSTRING("MP4 Video Decode")));
1.170 + NS_ENSURE_TRUE(mVideo.mTaskQueue, NS_ERROR_FAILURE);
1.171 +
1.172 + return NS_OK;
1.173 +}
1.174 +
1.175 +nsresult
1.176 +MP4Reader::ReadMetadata(MediaInfo* aInfo,
1.177 + MetadataTags** aTags)
1.178 +{
1.179 + bool ok = mDemuxer->Init();
1.180 + NS_ENSURE_TRUE(ok, NS_ERROR_FAILURE);
1.181 +
1.182 + const AudioDecoderConfig& audio = mDemuxer->AudioConfig();
1.183 + mInfo.mAudio.mHasAudio = mAudio.mActive = mDemuxer->HasAudio() &&
1.184 + audio.IsValidConfig();
1.185 + // If we have audio, we *only* allow AAC to be decoded.
1.186 + if (HasAudio() && audio.codec() != kCodecAAC) {
1.187 + return NS_ERROR_FAILURE;
1.188 + }
1.189 +
1.190 + const VideoDecoderConfig& video = mDemuxer->VideoConfig();
1.191 + mInfo.mVideo.mHasVideo = mVideo.mActive = mDemuxer->HasVideo() &&
1.192 + video.IsValidConfig();
1.193 + // If we have video, we *only* allow H.264 to be decoded.
1.194 + if (HasVideo() && video.codec() != kCodecH264) {
1.195 + return NS_ERROR_FAILURE;
1.196 + }
1.197 +
1.198 + mPlatform = PlatformDecoderModule::Create();
1.199 + NS_ENSURE_TRUE(mPlatform, NS_ERROR_FAILURE);
1.200 +
1.201 + if (HasAudio()) {
1.202 + mInfo.mAudio.mRate = audio.samples_per_second();
1.203 + mInfo.mAudio.mChannels = ChannelLayoutToChannelCount(audio.channel_layout());
1.204 + mAudio.mCallback = new DecoderCallback(this, kAudio);
1.205 + mAudio.mDecoder = mPlatform->CreateAACDecoder(audio,
1.206 + mAudio.mTaskQueue,
1.207 + mAudio.mCallback);
1.208 + NS_ENSURE_TRUE(mAudio.mDecoder != nullptr, NS_ERROR_FAILURE);
1.209 + nsresult rv = mAudio.mDecoder->Init();
1.210 + NS_ENSURE_SUCCESS(rv, rv);
1.211 + }
1.212 +
1.213 + if (HasVideo()) {
1.214 + IntSize sz = video.natural_size();
1.215 + mInfo.mVideo.mDisplay = nsIntSize(sz.width(), sz.height());
1.216 + mVideo.mCallback = new DecoderCallback(this, kVideo);
1.217 + mVideo.mDecoder = mPlatform->CreateH264Decoder(video,
1.218 + mLayersBackendType,
1.219 + mDecoder->GetImageContainer(),
1.220 + mVideo.mTaskQueue,
1.221 + mVideo.mCallback);
1.222 + NS_ENSURE_TRUE(mVideo.mDecoder != nullptr, NS_ERROR_FAILURE);
1.223 + nsresult rv = mVideo.mDecoder->Init();
1.224 + NS_ENSURE_SUCCESS(rv, rv);
1.225 + }
1.226 +
1.227 + // Get the duration, and report it to the decoder if we have it.
1.228 + Microseconds duration = mDemuxer->Duration();
1.229 + if (duration != -1) {
1.230 + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
1.231 + mDecoder->SetMediaDuration(duration);
1.232 + }
1.233 + // We can seek if we get a duration *and* the reader reports that it's
1.234 + // seekable.
1.235 + if (!mDemuxer->CanSeek()) {
1.236 + mDecoder->SetMediaSeekable(false);
1.237 + }
1.238 +
1.239 + *aInfo = mInfo;
1.240 + *aTags = nullptr;
1.241 +
1.242 + return NS_OK;
1.243 +}
1.244 +
1.245 +bool
1.246 +MP4Reader::HasAudio()
1.247 +{
1.248 + return mAudio.mActive;
1.249 +}
1.250 +
1.251 +bool
1.252 +MP4Reader::HasVideo()
1.253 +{
1.254 + return mVideo.mActive;
1.255 +}
1.256 +
1.257 +MP4Reader::DecoderData&
1.258 +MP4Reader::GetDecoderData(TrackType aTrack)
1.259 +{
1.260 + MOZ_ASSERT(aTrack == kAudio || aTrack == kVideo);
1.261 + return (aTrack == kAudio) ? mAudio : mVideo;
1.262 +}
1.263 +
1.264 +MP4SampleQueue&
1.265 +MP4Reader::SampleQueue(TrackType aTrack)
1.266 +{
1.267 + return GetDecoderData(aTrack).mDemuxedSamples;
1.268 +}
1.269 +
1.270 +MediaDataDecoder*
1.271 +MP4Reader::Decoder(mp4_demuxer::TrackType aTrack)
1.272 +{
1.273 + return GetDecoderData(aTrack).mDecoder;
1.274 +}
1.275 +
1.276 +MP4Sample*
1.277 +MP4Reader::PopSample(TrackType aTrack)
1.278 +{
1.279 + // Unfortunately the demuxer outputs in the order samples appear in the
1.280 + // media, not on a per stream basis. We cache the samples we get from
1.281 + // streams other than the one we want.
1.282 + MP4SampleQueue& sampleQueue = SampleQueue(aTrack);
1.283 + while (sampleQueue.empty()) {
1.284 + nsAutoPtr<MP4Sample> sample;
1.285 + bool eos = false;
1.286 + bool ok = mDemuxer->Demux(&sample, &eos);
1.287 + if (!ok || eos) {
1.288 + MOZ_ASSERT(!sample);
1.289 + return nullptr;
1.290 + }
1.291 + MOZ_ASSERT(sample);
1.292 + MP4Sample* s = sample.forget();
1.293 + SampleQueue(s->type).push_back(s);
1.294 + }
1.295 + MOZ_ASSERT(!sampleQueue.empty());
1.296 + MP4Sample* sample = sampleQueue.front();
1.297 + sampleQueue.pop_front();
1.298 + return sample;
1.299 +}
1.300 +
1.301 +// How async decoding works:
1.302 +//
1.303 +// When MP4Reader::Decode() is called:
1.304 +// * Lock the DecoderData. We assume the state machine wants
1.305 +// output from the decoder (in future, we'll assume decoder wants input
1.306 +// when the output MediaQueue isn't "full").
1.307 +// * Cache the value of mNumSamplesOutput, as prevFramesOutput.
1.308 +// * While we've not output data (mNumSamplesOutput != prevNumFramesOutput)
1.309 +// and while we still require input, we demux and input data in the reader.
1.310 +// We assume we require input if
1.311 +// ((mNumSamplesInput - mNumSamplesOutput) < sDecodeAheadMargin) or
1.312 +// mInputExhausted is true. Before we send input, we reset mInputExhausted
1.313 +// and increment mNumFrameInput, and drop the lock on DecoderData.
1.314 +// * Once we no longer require input, we wait on the DecoderData
1.315 +// lock for output, or for the input exhausted callback. If we receive the
1.316 +// input exhausted callback, we go back and input more data.
1.317 +// * When our output callback is called, we take the DecoderData lock and
1.318 +// increment mNumSamplesOutput. We notify the DecoderData lock. This will
1.319 +// awaken the Decode thread, and unblock it, and it will return.
1.320 +bool
1.321 +MP4Reader::Decode(TrackType aTrack)
1.322 +{
1.323 + DecoderData& data = GetDecoderData(aTrack);
1.324 + MOZ_ASSERT(data.mDecoder);
1.325 +
1.326 + data.mMonitor.Lock();
1.327 + uint64_t prevNumFramesOutput = data.mNumSamplesOutput;
1.328 + while (prevNumFramesOutput == data.mNumSamplesOutput) {
1.329 + data.mMonitor.AssertCurrentThreadOwns();
1.330 + if (data.mError) {
1.331 + // Decode error!
1.332 + data.mMonitor.Unlock();
1.333 + return false;
1.334 + }
1.335 + // Send input to the decoder, if we need to. We assume the decoder
1.336 + // needs input if it's told us it's out of input, or we're beneath the
1.337 + // "low water mark" for the amount of input we've sent it vs the amount
1.338 + // out output we've received. We always try to keep the decoder busy if
1.339 + // possible, so we try to maintain at least a few input samples ahead,
1.340 + // if we need output.
1.341 + while (prevNumFramesOutput == data.mNumSamplesOutput &&
1.342 + (data.mInputExhausted ||
1.343 + (data.mNumSamplesInput - data.mNumSamplesOutput) < data.mDecodeAhead)) {
1.344 + data.mMonitor.AssertCurrentThreadOwns();
1.345 + data.mMonitor.Unlock();
1.346 + nsAutoPtr<MP4Sample> compressed(PopSample(aTrack));
1.347 + if (!compressed) {
1.348 + // EOS, or error. Let the state machine know there are no more
1.349 + // frames coming.
1.350 + return false;
1.351 + }
1.352 + data.mMonitor.Lock();
1.353 + data.mInputExhausted = false;
1.354 + data.mNumSamplesInput++;
1.355 + data.mMonitor.Unlock();
1.356 + if (NS_FAILED(data.mDecoder->Input(compressed))) {
1.357 + return false;
1.358 + }
1.359 + // If Input() failed, we let the auto pointer delete |compressed|.
1.360 + // Otherwise, we assume the decoder will delete it when it's finished
1.361 + // with it.
1.362 + compressed.forget();
1.363 + data.mMonitor.Lock();
1.364 + }
1.365 + data.mMonitor.AssertCurrentThreadOwns();
1.366 + while (!data.mError &&
1.367 + prevNumFramesOutput == data.mNumSamplesOutput &&
1.368 + !data.mInputExhausted ) {
1.369 + data.mMonitor.Wait();
1.370 + }
1.371 + }
1.372 + data.mMonitor.AssertCurrentThreadOwns();
1.373 + data.mMonitor.Unlock();
1.374 + return true;
1.375 +}
1.376 +
1.377 +#ifdef LOG_SAMPLE_DECODE
1.378 +static const char*
1.379 +TrackTypeToStr(TrackType aTrack)
1.380 +{
1.381 + MOZ_ASSERT(aTrack == kAudio || aTrack == kVideo);
1.382 + switch (aTrack) {
1.383 + case kAudio: return "Audio";
1.384 + case kVideo: return "Video";
1.385 + default: return "Unknown";
1.386 + }
1.387 +}
1.388 +#endif
1.389 +
1.390 +void
1.391 +MP4Reader::Output(mp4_demuxer::TrackType aTrack, MediaData* aSample)
1.392 +{
1.393 +#ifdef LOG_SAMPLE_DECODE
1.394 + LOG("Decoded %s sample time=%lld dur=%lld",
1.395 + TrackTypeToStr(aTrack), aSample->mTime, aSample->mDuration);
1.396 +#endif
1.397 +
1.398 + DecoderData& data = GetDecoderData(aTrack);
1.399 + // Don't accept output while we're flushing.
1.400 + MonitorAutoLock mon(data.mMonitor);
1.401 + if (data.mIsFlushing) {
1.402 + mon.NotifyAll();
1.403 + return;
1.404 + }
1.405 +
1.406 + switch (aTrack) {
1.407 + case kAudio: {
1.408 + MOZ_ASSERT(aSample->mType == MediaData::AUDIO_SAMPLES);
1.409 + AudioQueue().Push(static_cast<AudioData*>(aSample));
1.410 + break;
1.411 + }
1.412 + case kVideo: {
1.413 + MOZ_ASSERT(aSample->mType == MediaData::VIDEO_FRAME);
1.414 + VideoQueue().Push(static_cast<VideoData*>(aSample));
1.415 + break;
1.416 + }
1.417 + default:
1.418 + break;
1.419 + }
1.420 +
1.421 + data.mNumSamplesOutput++;
1.422 + mon.NotifyAll();
1.423 +}
1.424 +
1.425 +void
1.426 +MP4Reader::InputExhausted(mp4_demuxer::TrackType aTrack)
1.427 +{
1.428 + DecoderData& data = GetDecoderData(aTrack);
1.429 + MonitorAutoLock mon(data.mMonitor);
1.430 + data.mInputExhausted = true;
1.431 + mon.NotifyAll();
1.432 +}
1.433 +
1.434 +void
1.435 +MP4Reader::Error(mp4_demuxer::TrackType aTrack)
1.436 +{
1.437 + DecoderData& data = GetDecoderData(aTrack);
1.438 + MonitorAutoLock mon(data.mMonitor);
1.439 + data.mError = true;
1.440 + mon.NotifyAll();
1.441 +}
1.442 +
1.443 +bool
1.444 +MP4Reader::DecodeAudioData()
1.445 +{
1.446 + MOZ_ASSERT(HasAudio() && mPlatform && mAudio.mDecoder);
1.447 + return Decode(kAudio);
1.448 +}
1.449 +
1.450 +void
1.451 +MP4Reader::Flush(mp4_demuxer::TrackType aTrack)
1.452 +{
1.453 + DecoderData& data = GetDecoderData(aTrack);
1.454 + if (!data.mDecoder) {
1.455 + return;
1.456 + }
1.457 + // Purge the current decoder's state.
1.458 + // Set a flag so that we ignore all output while we call
1.459 + // MediaDataDecoder::Flush().
1.460 + {
1.461 + data.mIsFlushing = true;
1.462 + MonitorAutoLock mon(data.mMonitor);
1.463 + }
1.464 + data.mDecoder->Flush();
1.465 + {
1.466 + data.mIsFlushing = false;
1.467 + MonitorAutoLock mon(data.mMonitor);
1.468 + }
1.469 +}
1.470 +
1.471 +bool
1.472 +MP4Reader::SkipVideoDemuxToNextKeyFrame(int64_t aTimeThreshold, uint32_t& parsed)
1.473 +{
1.474 + MOZ_ASSERT(mVideo.mDecoder);
1.475 +
1.476 + Flush(kVideo);
1.477 +
1.478 + // Loop until we reach the next keyframe after the threshold.
1.479 + while (true) {
1.480 + nsAutoPtr<MP4Sample> compressed(PopSample(kVideo));
1.481 + if (!compressed) {
1.482 + // EOS, or error. Let the state machine know.
1.483 + return false;
1.484 + }
1.485 + parsed++;
1.486 + if (!compressed->is_sync_point ||
1.487 + compressed->composition_timestamp < aTimeThreshold) {
1.488 + continue;
1.489 + }
1.490 + mVideo.mDemuxedSamples.push_front(compressed.forget());
1.491 + break;
1.492 + }
1.493 +
1.494 + return true;
1.495 +}
1.496 +
1.497 +bool
1.498 +MP4Reader::DecodeVideoFrame(bool &aKeyframeSkip,
1.499 + int64_t aTimeThreshold)
1.500 +{
1.501 + // Record number of frames decoded and parsed. Automatically update the
1.502 + // stats counters using the AutoNotifyDecoded stack-based class.
1.503 + uint32_t parsed = 0, decoded = 0;
1.504 + AbstractMediaDecoder::AutoNotifyDecoded autoNotify(mDecoder, parsed, decoded);
1.505 +
1.506 + MOZ_ASSERT(HasVideo() && mPlatform && mVideo.mDecoder);
1.507 +
1.508 + if (aKeyframeSkip) {
1.509 + bool ok = SkipVideoDemuxToNextKeyFrame(aTimeThreshold, parsed);
1.510 + if (!ok) {
1.511 + NS_WARNING("Failed to skip demux up to next keyframe");
1.512 + return false;
1.513 + }
1.514 + aKeyframeSkip = false;
1.515 + nsresult rv = mVideo.mDecoder->Flush();
1.516 + NS_ENSURE_SUCCESS(rv, false);
1.517 + }
1.518 +
1.519 + bool rv = Decode(kVideo);
1.520 + {
1.521 + // Report the number of "decoded" frames as the difference in the
1.522 + // mNumSamplesOutput field since the last time we were called.
1.523 + MonitorAutoLock mon(mVideo.mMonitor);
1.524 + uint64_t delta = mVideo.mNumSamplesOutput - mLastReportedNumDecodedFrames;
1.525 + decoded = static_cast<uint32_t>(delta);
1.526 + mLastReportedNumDecodedFrames = mVideo.mNumSamplesOutput;
1.527 + }
1.528 + return rv;
1.529 +}
1.530 +
1.531 +nsresult
1.532 +MP4Reader::Seek(int64_t aTime,
1.533 + int64_t aStartTime,
1.534 + int64_t aEndTime,
1.535 + int64_t aCurrentTime)
1.536 +{
1.537 + if (!mDemuxer->CanSeek()) {
1.538 + return NS_ERROR_FAILURE;
1.539 + }
1.540 + return NS_ERROR_NOT_IMPLEMENTED;
1.541 +}
1.542 +
1.543 +} // namespace mozilla
