michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* vim:set ts=2 sw=2 sts=2 et cindent: */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: #include "MediaPluginReader.h"
michael@0: #include "mozilla/TimeStamp.h"
michael@0: #include "mozilla/dom/TimeRanges.h"
michael@0: #include "mozilla/gfx/Point.h"
michael@0: #include "MediaResource.h"
michael@0: #include "VideoUtils.h"
michael@0: #include "MediaPluginDecoder.h"
michael@0: #include "MediaPluginHost.h"
michael@0: #include "MediaDecoderStateMachine.h"
michael@0: #include "ImageContainer.h"
michael@0: #include "AbstractMediaDecoder.h"
michael@0: #include "gfx2DGlue.h"
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: using namespace mozilla::gfx;
michael@0: 
michael@0: typedef mozilla::layers::Image Image;
michael@0: typedef mozilla::layers::PlanarYCbCrImage PlanarYCbCrImage;
michael@0: 
michael@0: MediaPluginReader::MediaPluginReader(AbstractMediaDecoder *aDecoder,
michael@0:                                      const nsACString& aContentType) :
michael@0:   MediaDecoderReader(aDecoder),
michael@0:   mType(aContentType),
michael@0:   mPlugin(nullptr),
michael@0:   mHasAudio(false),
michael@0:   mHasVideo(false),
michael@0:   mVideoSeekTimeUs(-1),
michael@0:   mAudioSeekTimeUs(-1)
michael@0: {
michael@0: }
michael@0: 
michael@0: MediaPluginReader::~MediaPluginReader()
michael@0: {
michael@0:   ResetDecode();
michael@0: }
michael@0: 
michael@0: nsresult MediaPluginReader::Init(MediaDecoderReader* aCloneDonor)
michael@0: {
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult MediaPluginReader::ReadMetadata(MediaInfo* aInfo,
michael@0:                                          MetadataTags** aTags)
michael@0: {
michael@0:   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
michael@0: 
michael@0:   if (!mPlugin) {
michael@0:     mPlugin = GetMediaPluginHost()->CreateDecoder(mDecoder->GetResource(), mType);
michael@0:     if (!mPlugin) {
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Set the total duration (the max of the audio and video track).
michael@0:   int64_t durationUs;
michael@0:   mPlugin->GetDuration(mPlugin, &durationUs);
michael@0:   if (durationUs) {
michael@0:     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
michael@0:     mDecoder->SetMediaDuration(durationUs);
michael@0:   }
michael@0: 
michael@0:   if (mPlugin->HasVideo(mPlugin)) {
michael@0:     int32_t width, height;
michael@0:     mPlugin->GetVideoParameters(mPlugin, &width, &height);
michael@0:     nsIntRect pictureRect(0, 0, width, height);
michael@0: 
michael@0:     // Validate the container-reported frame and pictureRect sizes. This ensures
michael@0:     // that our video frame creation code doesn't overflow.
michael@0:     nsIntSize displaySize(width, height);
michael@0:     nsIntSize frameSize(width, height);
michael@0:     if (!IsValidVideoRegion(frameSize, pictureRect, displaySize)) {
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0: 
michael@0:     // Video track's frame sizes will not overflow. Activate the video track.
michael@0:     mHasVideo = mInfo.mVideo.mHasVideo = true;
michael@0:     mInfo.mVideo.mDisplay = displaySize;
michael@0:     mPicture = pictureRect;
michael@0:     mInitialFrame = frameSize;
michael@0:     VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
michael@0:     if (container) {
michael@0:       container->SetCurrentFrame(gfxIntSize(displaySize.width, displaySize.height),
michael@0:                                  nullptr,
michael@0:                                  mozilla::TimeStamp::Now());
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   if (mPlugin->HasAudio(mPlugin)) {
michael@0:     int32_t numChannels, sampleRate;
michael@0:     mPlugin->GetAudioParameters(mPlugin, &numChannels, &sampleRate);
michael@0:     mHasAudio = mInfo.mAudio.mHasAudio = true;
michael@0:     mInfo.mAudio.mChannels = numChannels;
michael@0:     mInfo.mAudio.mRate = sampleRate;
michael@0:   }
michael@0: 
michael@0:  *aInfo = mInfo;
michael@0:  *aTags = nullptr;
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: // Resets all state related to decoding, emptying all buffers etc.
michael@0: nsresult MediaPluginReader::ResetDecode()
michael@0: {
michael@0:   if (mLastVideoFrame) {
michael@0:     mLastVideoFrame = nullptr;
michael@0:   }
michael@0:   if (mPlugin) {
michael@0:     GetMediaPluginHost()->DestroyDecoder(mPlugin);
michael@0:     mPlugin = nullptr;
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: bool MediaPluginReader::DecodeVideoFrame(bool &aKeyframeSkip,
michael@0:                                          int64_t aTimeThreshold)
michael@0: {
michael@0:   // Record number of frames decoded and parsed. Automatically update the
michael@0:   // stats counters using the AutoNotifyDecoded stack-based class.
michael@0:   uint32_t parsed = 0, decoded = 0;
michael@0:   AbstractMediaDecoder::AutoNotifyDecoded autoNotify(mDecoder, parsed, decoded);
michael@0: 
michael@0:   // Throw away the currently buffered frame if we are seeking.
michael@0:   if (mLastVideoFrame && mVideoSeekTimeUs != -1) {
michael@0:     mLastVideoFrame = nullptr;
michael@0:   }
michael@0: 
michael@0:   ImageBufferCallback bufferCallback(mDecoder->GetImageContainer());
michael@0:   nsRefPtr<Image> currentImage;
michael@0: 
michael@0:   // Read next frame
michael@0:   while (true) {
michael@0:     MPAPI::VideoFrame frame;
michael@0:     if (!mPlugin->ReadVideo(mPlugin, &frame, mVideoSeekTimeUs, &bufferCallback)) {
michael@0:       // We reached the end of the video stream. If we have a buffered
michael@0:       // video frame, push it the video queue using the total duration
michael@0:       // of the video as the end time.
michael@0:       if (mLastVideoFrame) {
michael@0:         int64_t durationUs;
michael@0:         mPlugin->GetDuration(mPlugin, &durationUs);
michael@0:         durationUs = std::max<int64_t>(durationUs - mLastVideoFrame->mTime, 0);
michael@0:         mVideoQueue.Push(VideoData::ShallowCopyUpdateDuration(mLastVideoFrame,
michael@0:                                                               durationUs));
michael@0:         mLastVideoFrame = nullptr;
michael@0:       }
michael@0:       return false;
michael@0:     }
michael@0:     mVideoSeekTimeUs = -1;
michael@0: 
michael@0:     if (aKeyframeSkip) {
michael@0:       // Disable keyframe skipping for now as
michael@0:       // stagefright doesn't seem to be telling us
michael@0:       // when a frame is a keyframe.
michael@0: #if 0
michael@0:       if (!frame.mKeyFrame) {
michael@0:         ++parsed;
michael@0:         continue;
michael@0:       }
michael@0: #endif
michael@0:       aKeyframeSkip = false;
michael@0:     }
michael@0: 
michael@0:     if (frame.mSize == 0)
michael@0:       return true;
michael@0: 
michael@0:     currentImage = bufferCallback.GetImage();
michael@0:     int64_t pos = mDecoder->GetResource()->Tell();
michael@0:     IntRect picture = ToIntRect(mPicture);
michael@0: 
michael@0:     nsAutoPtr<VideoData> v;
michael@0:     if (currentImage) {
michael@0:       gfx::IntSize frameSize = currentImage->GetSize();
michael@0:       if (frameSize.width != mInitialFrame.width ||
michael@0:           frameSize.height != mInitialFrame.height) {
michael@0:         // Frame size is different from what the container reports. This is legal,
michael@0:         // and we will preserve the ratio of the crop rectangle as it
michael@0:         // was reported relative to the picture size reported by the container.
michael@0:         picture.x = (mPicture.x * frameSize.width) / mInitialFrame.width;
michael@0:         picture.y = (mPicture.y * frameSize.height) / mInitialFrame.height;
michael@0:         picture.width = (frameSize.width * mPicture.width) / mInitialFrame.width;
michael@0:         picture.height = (frameSize.height * mPicture.height) / mInitialFrame.height;
michael@0:       }
michael@0: 
michael@0:       v = VideoData::CreateFromImage(mInfo.mVideo,
michael@0:                                      mDecoder->GetImageContainer(),
michael@0:                                      pos,
michael@0:                                      frame.mTimeUs,
michael@0:                                      1, // We don't know the duration yet.
michael@0:                                      currentImage,
michael@0:                                      frame.mKeyFrame,
michael@0:                                      -1,
michael@0:                                      picture);
michael@0:     } else {
michael@0:       // Assume YUV
michael@0:       VideoData::YCbCrBuffer b;
michael@0:       b.mPlanes[0].mData = static_cast<uint8_t *>(frame.Y.mData);
michael@0:       b.mPlanes[0].mStride = frame.Y.mStride;
michael@0:       b.mPlanes[0].mHeight = frame.Y.mHeight;
michael@0:       b.mPlanes[0].mWidth = frame.Y.mWidth;
michael@0:       b.mPlanes[0].mOffset = frame.Y.mOffset;
michael@0:       b.mPlanes[0].mSkip = frame.Y.mSkip;
michael@0: 
michael@0:       b.mPlanes[1].mData = static_cast<uint8_t *>(frame.Cb.mData);
michael@0:       b.mPlanes[1].mStride = frame.Cb.mStride;
michael@0:       b.mPlanes[1].mHeight = frame.Cb.mHeight;
michael@0:       b.mPlanes[1].mWidth = frame.Cb.mWidth;
michael@0:       b.mPlanes[1].mOffset = frame.Cb.mOffset;
michael@0:       b.mPlanes[1].mSkip = frame.Cb.mSkip;
michael@0: 
michael@0:       b.mPlanes[2].mData = static_cast<uint8_t *>(frame.Cr.mData);
michael@0:       b.mPlanes[2].mStride = frame.Cr.mStride;
michael@0:       b.mPlanes[2].mHeight = frame.Cr.mHeight;
michael@0:       b.mPlanes[2].mWidth = frame.Cr.mWidth;
michael@0:       b.mPlanes[2].mOffset = frame.Cr.mOffset;
michael@0:       b.mPlanes[2].mSkip = frame.Cr.mSkip;
michael@0: 
michael@0:       if (frame.Y.mWidth != mInitialFrame.width ||
michael@0:           frame.Y.mHeight != mInitialFrame.height) {
michael@0: 
michael@0:         // Frame size is different from what the container reports. This is legal,
michael@0:         // and we will preserve the ratio of the crop rectangle as it
michael@0:         // was reported relative to the picture size reported by the container.
michael@0:         picture.x = (mPicture.x * frame.Y.mWidth) / mInitialFrame.width;
michael@0:         picture.y = (mPicture.y * frame.Y.mHeight) / mInitialFrame.height;
michael@0:         picture.width = (frame.Y.mWidth * mPicture.width) / mInitialFrame.width;
michael@0:         picture.height = (frame.Y.mHeight * mPicture.height) / mInitialFrame.height;
michael@0:       }
michael@0: 
michael@0:       // This is the approximate byte position in the stream.
michael@0:       v = VideoData::Create(mInfo.mVideo,
michael@0:                             mDecoder->GetImageContainer(),
michael@0:                             pos,
michael@0:                             frame.mTimeUs,
michael@0:                             1, // We don't know the duration yet.
michael@0:                             b,
michael@0:                             frame.mKeyFrame,
michael@0:                             -1,
michael@0:                             picture);
michael@0:     }
michael@0:  
michael@0:     if (!v) {
michael@0:       return false;
michael@0:     }
michael@0:     parsed++;
michael@0:     decoded++;
michael@0:     NS_ASSERTION(decoded <= parsed, "Expect to decode fewer frames than parsed in MediaPlugin...");
michael@0: 
michael@0:     // Since MPAPI doesn't give us the end time of frames, we keep one frame
michael@0:     // buffered in MediaPluginReader and push it into the queue as soon
michael@0:     // we read the following frame so we can use that frame's start time as
michael@0:     // the end time of the buffered frame.
michael@0:     if (!mLastVideoFrame) {
michael@0:       mLastVideoFrame = v;
michael@0:       continue;
michael@0:     }
michael@0: 
michael@0:     // Calculate the duration as the timestamp of the current frame minus the
michael@0:     // timestamp of the previous frame. We can then return the previously
michael@0:     // decoded frame, and it will have a valid timestamp.
michael@0:     int64_t duration = v->mTime - mLastVideoFrame->mTime;
michael@0:     mLastVideoFrame = VideoData::ShallowCopyUpdateDuration(mLastVideoFrame, duration);
michael@0: 
michael@0:     // We have the start time of the next frame, so we can push the previous
michael@0:     // frame into the queue, except if the end time is below the threshold,
michael@0:     // in which case it wouldn't be displayed anyway.
michael@0:     if (mLastVideoFrame->GetEndTime() < aTimeThreshold) {
michael@0:       mLastVideoFrame = nullptr;
michael@0:       continue;
michael@0:     }
michael@0: 
michael@0:     mVideoQueue.Push(mLastVideoFrame.forget());
michael@0: 
michael@0:     // Buffer the current frame we just decoded.
michael@0:     mLastVideoFrame = v;
michael@0: 
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   return true;
michael@0: }
michael@0: 
michael@0: bool MediaPluginReader::DecodeAudioData()
michael@0: {
michael@0:   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
michael@0: 
michael@0:   // This is the approximate byte position in the stream.
michael@0:   int64_t pos = mDecoder->GetResource()->Tell();
michael@0: 
michael@0:   // Read next frame
michael@0:   MPAPI::AudioFrame source;
michael@0:   if (!mPlugin->ReadAudio(mPlugin, &source, mAudioSeekTimeUs)) {
michael@0:     return false;
michael@0:   }
michael@0:   mAudioSeekTimeUs = -1;
michael@0: 
michael@0:   // Ignore empty buffers which stagefright media read will sporadically return
michael@0:   if (source.mSize == 0)
michael@0:     return true;
michael@0: 
michael@0:   uint32_t frames = source.mSize / (source.mAudioChannels *
michael@0:                                     sizeof(AudioDataValue));
michael@0: 
michael@0:   typedef AudioCompactor::NativeCopy MPCopy;
michael@0:   return mAudioCompactor.Push(pos,
michael@0:                               source.mTimeUs,
michael@0:                               source.mAudioSampleRate,
michael@0:                               frames,
michael@0:                               source.mAudioChannels,
michael@0:                               MPCopy(static_cast<uint8_t *>(source.mData),
michael@0:                                      source.mSize,
michael@0:                                      source.mAudioChannels));
michael@0: }
michael@0: 
michael@0: nsresult MediaPluginReader::Seek(int64_t aTarget, int64_t aStartTime, int64_t aEndTime, int64_t aCurrentTime)
michael@0: {
michael@0:   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
michael@0: 
michael@0:   mVideoQueue.Reset();
michael@0:   mAudioQueue.Reset();
michael@0: 
michael@0:   mAudioSeekTimeUs = mVideoSeekTimeUs = aTarget;
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: MediaPluginReader::ImageBufferCallback::ImageBufferCallback(mozilla::layers::ImageContainer *aImageContainer) :
michael@0:   mImageContainer(aImageContainer)
michael@0: {
michael@0: }
michael@0: 
michael@0: void *
michael@0: MediaPluginReader::ImageBufferCallback::operator()(size_t aWidth, size_t aHeight,
michael@0:                                                    MPAPI::ColorFormat aColorFormat)
michael@0: {
michael@0:   if (!mImageContainer) {
michael@0:     NS_WARNING("No image container to construct an image");
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   nsRefPtr<Image> image;
michael@0:   switch(aColorFormat) {
michael@0:     case MPAPI::RGB565:
michael@0:       image = mozilla::layers::CreateSharedRGBImage(mImageContainer,
michael@0:                                                     nsIntSize(aWidth, aHeight),
michael@0:                                                     gfxImageFormat::RGB16_565);
michael@0:       if (!image) {
michael@0:         NS_WARNING("Could not create rgb image");
michael@0:         return nullptr;
michael@0:       }
michael@0: 
michael@0:       mImage = image;
michael@0:       return image->AsSharedImage()->GetBuffer();
michael@0:     case MPAPI::I420:
michael@0:       return CreateI420Image(aWidth, aHeight);
michael@0:     default:
michael@0:       NS_NOTREACHED("Color format not supported");
michael@0:       return nullptr;
michael@0:   }
michael@0: }
michael@0: 
michael@0: uint8_t *
michael@0: MediaPluginReader::ImageBufferCallback::CreateI420Image(size_t aWidth,
michael@0:                                                         size_t aHeight)
michael@0: {
michael@0:   mImage = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR);
michael@0:   PlanarYCbCrImage *yuvImage = static_cast<PlanarYCbCrImage *>(mImage.get());
michael@0: 
michael@0:   if (!yuvImage) {
michael@0:     NS_WARNING("Could not create I420 image");
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   size_t frameSize = aWidth * aHeight;
michael@0: 
michael@0:   // Allocate enough for one full resolution Y plane
michael@0:   // and two quarter resolution Cb/Cr planes.
michael@0:   uint8_t *buffer = yuvImage->AllocateAndGetNewBuffer(frameSize * 3 / 2);
michael@0: 
michael@0:   mozilla::layers::PlanarYCbCrData frameDesc;
michael@0: 
michael@0:   frameDesc.mYChannel = buffer;
michael@0:   frameDesc.mCbChannel = buffer + frameSize;
michael@0:   frameDesc.mCrChannel = buffer + frameSize * 5 / 4;
michael@0: 
michael@0:   frameDesc.mYSize = IntSize(aWidth, aHeight);
michael@0:   frameDesc.mCbCrSize = IntSize(aWidth / 2, aHeight / 2);
michael@0: 
michael@0:   frameDesc.mYStride = aWidth;
michael@0:   frameDesc.mCbCrStride = aWidth / 2;
michael@0: 
michael@0:   frameDesc.mYSkip = 0;
michael@0:   frameDesc.mCbSkip = 0;
michael@0:   frameDesc.mCrSkip = 0;
michael@0: 
michael@0:   frameDesc.mPicX = 0;
michael@0:   frameDesc.mPicY = 0;
michael@0:   frameDesc.mPicSize = IntSize(aWidth, aHeight);
michael@0: 
michael@0:   yuvImage->SetDataNoCopy(frameDesc);
michael@0: 
michael@0:   return buffer;
michael@0: }
michael@0: 
michael@0: already_AddRefed<Image>
michael@0: MediaPluginReader::ImageBufferCallback::GetImage()
michael@0: {
michael@0:   return mImage.forget();
michael@0: }
michael@0: 
michael@0: } // namespace mozilla