The Tor Browser: content/media/plugins/MediaPluginReader.cpp@97036ab72558

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

     1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */

     2 /* vim:set ts=2 sw=2 sts=2 et cindent: */

     3 /* This Source Code Form is subject to the terms of the Mozilla Public

     4  * License, v. 2.0. If a copy of the MPL was not distributed with this file,

     5  * You can obtain one at http://mozilla.org/MPL/2.0/. */

     6 #include "MediaPluginReader.h"

     7 #include "mozilla/TimeStamp.h"

     8 #include "mozilla/dom/TimeRanges.h"

     9 #include "mozilla/gfx/Point.h"

    10 #include "MediaResource.h"

    11 #include "VideoUtils.h"

    12 #include "MediaPluginDecoder.h"

    13 #include "MediaPluginHost.h"

    14 #include "MediaDecoderStateMachine.h"

    15 #include "ImageContainer.h"

    16 #include "AbstractMediaDecoder.h"

    17 #include "gfx2DGlue.h"

    19 namespace mozilla {

    21 using namespace mozilla::gfx;

    23 typedef mozilla::layers::Image Image;

    24 typedef mozilla::layers::PlanarYCbCrImage PlanarYCbCrImage;

    26 MediaPluginReader::MediaPluginReader(AbstractMediaDecoder *aDecoder,

    27                                      const nsACString& aContentType) :

    28   MediaDecoderReader(aDecoder),

    29   mType(aContentType),

    30   mPlugin(nullptr),

    31   mHasAudio(false),

    32   mHasVideo(false),

    33   mVideoSeekTimeUs(-1),

    34   mAudioSeekTimeUs(-1)

    35 {

    36 }

    38 MediaPluginReader::~MediaPluginReader()

    39 {

    40   ResetDecode();

    41 }

    43 nsresult MediaPluginReader::Init(MediaDecoderReader* aCloneDonor)

    44 {

    45   return NS_OK;

    46 }

    48 nsresult MediaPluginReader::ReadMetadata(MediaInfo* aInfo,

    49                                          MetadataTags** aTags)

    50 {

    51   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");

    53   if (!mPlugin) {

    54     mPlugin = GetMediaPluginHost()->CreateDecoder(mDecoder->GetResource(), mType);

    55     if (!mPlugin) {

    56       return NS_ERROR_FAILURE;

    57     }

    58   }

    60   // Set the total duration (the max of the audio and video track).

    61   int64_t durationUs;

    62   mPlugin->GetDuration(mPlugin, &durationUs);

    63   if (durationUs) {

    64     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());

    65     mDecoder->SetMediaDuration(durationUs);

    66   }

    68   if (mPlugin->HasVideo(mPlugin)) {

    69     int32_t width, height;

    70     mPlugin->GetVideoParameters(mPlugin, &width, &height);

    71     nsIntRect pictureRect(0, 0, width, height);

    73     // Validate the container-reported frame and pictureRect sizes. This ensures

    74     // that our video frame creation code doesn't overflow.

    75     nsIntSize displaySize(width, height);

    76     nsIntSize frameSize(width, height);

    77     if (!IsValidVideoRegion(frameSize, pictureRect, displaySize)) {

    78       return NS_ERROR_FAILURE;

    79     }

    81     // Video track's frame sizes will not overflow. Activate the video track.

    82     mHasVideo = mInfo.mVideo.mHasVideo = true;

    83     mInfo.mVideo.mDisplay = displaySize;

    84     mPicture = pictureRect;

    85     mInitialFrame = frameSize;

    86     VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();

    87     if (container) {

    88       container->SetCurrentFrame(gfxIntSize(displaySize.width, displaySize.height),

    89                                  nullptr,

    90                                  mozilla::TimeStamp::Now());

    91     }

    92   }

    94   if (mPlugin->HasAudio(mPlugin)) {

    95     int32_t numChannels, sampleRate;

    96     mPlugin->GetAudioParameters(mPlugin, &numChannels, &sampleRate);

    97     mHasAudio = mInfo.mAudio.mHasAudio = true;

    98     mInfo.mAudio.mChannels = numChannels;

    99     mInfo.mAudio.mRate = sampleRate;

   100   }

   102  *aInfo = mInfo;

   103  *aTags = nullptr;

   104   return NS_OK;

   105 }

   107 // Resets all state related to decoding, emptying all buffers etc.

   108 nsresult MediaPluginReader::ResetDecode()

   109 {

   110   if (mLastVideoFrame) {

   111     mLastVideoFrame = nullptr;

   112   }

   113   if (mPlugin) {

   114     GetMediaPluginHost()->DestroyDecoder(mPlugin);

   115     mPlugin = nullptr;

   116   }

   118   return NS_OK;

   119 }

   121 bool MediaPluginReader::DecodeVideoFrame(bool &aKeyframeSkip,

   122                                          int64_t aTimeThreshold)

   123 {

   124   // Record number of frames decoded and parsed. Automatically update the

   125   // stats counters using the AutoNotifyDecoded stack-based class.

   126   uint32_t parsed = 0, decoded = 0;

   127   AbstractMediaDecoder::AutoNotifyDecoded autoNotify(mDecoder, parsed, decoded);

   129   // Throw away the currently buffered frame if we are seeking.

   130   if (mLastVideoFrame && mVideoSeekTimeUs != -1) {

   131     mLastVideoFrame = nullptr;

   132   }

   134   ImageBufferCallback bufferCallback(mDecoder->GetImageContainer());

   135   nsRefPtr<Image> currentImage;

   137   // Read next frame

   138   while (true) {

   139     MPAPI::VideoFrame frame;

   140     if (!mPlugin->ReadVideo(mPlugin, &frame, mVideoSeekTimeUs, &bufferCallback)) {

   141       // We reached the end of the video stream. If we have a buffered

   142       // video frame, push it the video queue using the total duration

   143       // of the video as the end time.

   144       if (mLastVideoFrame) {

   145         int64_t durationUs;

   146         mPlugin->GetDuration(mPlugin, &durationUs);

   147         durationUs = std::max<int64_t>(durationUs - mLastVideoFrame->mTime, 0);

   148         mVideoQueue.Push(VideoData::ShallowCopyUpdateDuration(mLastVideoFrame,

   149                                                               durationUs));

   150         mLastVideoFrame = nullptr;

   151       }

   152       return false;

   153     }

   154     mVideoSeekTimeUs = -1;

   156     if (aKeyframeSkip) {

   157       // Disable keyframe skipping for now as

   158       // stagefright doesn't seem to be telling us

   159       // when a frame is a keyframe.

   160 #if 0

   161       if (!frame.mKeyFrame) {

   162         ++parsed;

   163         continue;

   164       }

   165 #endif

   166       aKeyframeSkip = false;

   167     }

   169     if (frame.mSize == 0)

   170       return true;

   172     currentImage = bufferCallback.GetImage();

   173     int64_t pos = mDecoder->GetResource()->Tell();

   174     IntRect picture = ToIntRect(mPicture);

   176     nsAutoPtr<VideoData> v;

   177     if (currentImage) {

   178       gfx::IntSize frameSize = currentImage->GetSize();

   179       if (frameSize.width != mInitialFrame.width ||

   180           frameSize.height != mInitialFrame.height) {

   181         // Frame size is different from what the container reports. This is legal,

   182         // and we will preserve the ratio of the crop rectangle as it

   183         // was reported relative to the picture size reported by the container.

   184         picture.x = (mPicture.x * frameSize.width) / mInitialFrame.width;

   185         picture.y = (mPicture.y * frameSize.height) / mInitialFrame.height;

   186         picture.width = (frameSize.width * mPicture.width) / mInitialFrame.width;

   187         picture.height = (frameSize.height * mPicture.height) / mInitialFrame.height;

   188       }

   190       v = VideoData::CreateFromImage(mInfo.mVideo,

   191                                      mDecoder->GetImageContainer(),

   192                                      pos,

   193                                      frame.mTimeUs,

   194                                      1, // We don't know the duration yet.

   195                                      currentImage,

   196                                      frame.mKeyFrame,

   197                                      -1,

   198                                      picture);

   199     } else {

   200       // Assume YUV

   201       VideoData::YCbCrBuffer b;

   202       b.mPlanes[0].mData = static_cast<uint8_t *>(frame.Y.mData);

   203       b.mPlanes[0].mStride = frame.Y.mStride;

   204       b.mPlanes[0].mHeight = frame.Y.mHeight;

   205       b.mPlanes[0].mWidth = frame.Y.mWidth;

   206       b.mPlanes[0].mOffset = frame.Y.mOffset;

   207       b.mPlanes[0].mSkip = frame.Y.mSkip;

   209       b.mPlanes[1].mData = static_cast<uint8_t *>(frame.Cb.mData);

   210       b.mPlanes[1].mStride = frame.Cb.mStride;

   211       b.mPlanes[1].mHeight = frame.Cb.mHeight;

   212       b.mPlanes[1].mWidth = frame.Cb.mWidth;

   213       b.mPlanes[1].mOffset = frame.Cb.mOffset;

   214       b.mPlanes[1].mSkip = frame.Cb.mSkip;

   216       b.mPlanes[2].mData = static_cast<uint8_t *>(frame.Cr.mData);

   217       b.mPlanes[2].mStride = frame.Cr.mStride;

   218       b.mPlanes[2].mHeight = frame.Cr.mHeight;

   219       b.mPlanes[2].mWidth = frame.Cr.mWidth;

   220       b.mPlanes[2].mOffset = frame.Cr.mOffset;

   221       b.mPlanes[2].mSkip = frame.Cr.mSkip;

   223       if (frame.Y.mWidth != mInitialFrame.width ||

   224           frame.Y.mHeight != mInitialFrame.height) {

   226         // Frame size is different from what the container reports. This is legal,

   227         // and we will preserve the ratio of the crop rectangle as it

   228         // was reported relative to the picture size reported by the container.

   229         picture.x = (mPicture.x * frame.Y.mWidth) / mInitialFrame.width;

   230         picture.y = (mPicture.y * frame.Y.mHeight) / mInitialFrame.height;

   231         picture.width = (frame.Y.mWidth * mPicture.width) / mInitialFrame.width;

   232         picture.height = (frame.Y.mHeight * mPicture.height) / mInitialFrame.height;

   233       }

   235       // This is the approximate byte position in the stream.

   236       v = VideoData::Create(mInfo.mVideo,

   237                             mDecoder->GetImageContainer(),

   238                             pos,

   239                             frame.mTimeUs,

   240                             1, // We don't know the duration yet.

   241                             b,

   242                             frame.mKeyFrame,

   243                             -1,

   244                             picture);

   245     }

   247     if (!v) {

   248       return false;

   249     }

   250     parsed++;

   251     decoded++;

   252     NS_ASSERTION(decoded <= parsed, "Expect to decode fewer frames than parsed in MediaPlugin...");

   254     // Since MPAPI doesn't give us the end time of frames, we keep one frame

   255     // buffered in MediaPluginReader and push it into the queue as soon

   256     // we read the following frame so we can use that frame's start time as

   257     // the end time of the buffered frame.

   258     if (!mLastVideoFrame) {

   259       mLastVideoFrame = v;

   260       continue;

   261     }

   263     // Calculate the duration as the timestamp of the current frame minus the

   264     // timestamp of the previous frame. We can then return the previously

   265     // decoded frame, and it will have a valid timestamp.

   266     int64_t duration = v->mTime - mLastVideoFrame->mTime;

   267     mLastVideoFrame = VideoData::ShallowCopyUpdateDuration(mLastVideoFrame, duration);

   269     // We have the start time of the next frame, so we can push the previous

   270     // frame into the queue, except if the end time is below the threshold,

   271     // in which case it wouldn't be displayed anyway.

   272     if (mLastVideoFrame->GetEndTime() < aTimeThreshold) {

   273       mLastVideoFrame = nullptr;

   274       continue;

   275     }

   277     mVideoQueue.Push(mLastVideoFrame.forget());

   279     // Buffer the current frame we just decoded.

   280     mLastVideoFrame = v;

   282     break;

   283   }

   285   return true;

   286 }

   288 bool MediaPluginReader::DecodeAudioData()

   289 {

   290   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");

   292   // This is the approximate byte position in the stream.

   293   int64_t pos = mDecoder->GetResource()->Tell();

   295   // Read next frame

   296   MPAPI::AudioFrame source;

   297   if (!mPlugin->ReadAudio(mPlugin, &source, mAudioSeekTimeUs)) {

   298     return false;

   299   }

   300   mAudioSeekTimeUs = -1;

   302   // Ignore empty buffers which stagefright media read will sporadically return

   303   if (source.mSize == 0)

   304     return true;

   306   uint32_t frames = source.mSize / (source.mAudioChannels *

   307                                     sizeof(AudioDataValue));

   309   typedef AudioCompactor::NativeCopy MPCopy;

   310   return mAudioCompactor.Push(pos,

   311                               source.mTimeUs,

   312                               source.mAudioSampleRate,

   313                               frames,

   314                               source.mAudioChannels,

   315                               MPCopy(static_cast<uint8_t *>(source.mData),

   316                                      source.mSize,

   317                                      source.mAudioChannels));

   318 }

   320 nsresult MediaPluginReader::Seek(int64_t aTarget, int64_t aStartTime, int64_t aEndTime, int64_t aCurrentTime)

   321 {

   322   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");

   324   mVideoQueue.Reset();

   325   mAudioQueue.Reset();

   327   mAudioSeekTimeUs = mVideoSeekTimeUs = aTarget;

   329   return NS_OK;

   330 }

   332 MediaPluginReader::ImageBufferCallback::ImageBufferCallback(mozilla::layers::ImageContainer *aImageContainer) :

   333   mImageContainer(aImageContainer)

   334 {

   335 }

   337 void *

   338 MediaPluginReader::ImageBufferCallback::operator()(size_t aWidth, size_t aHeight,

   339                                                    MPAPI::ColorFormat aColorFormat)

   340 {

   341   if (!mImageContainer) {

   342     NS_WARNING("No image container to construct an image");

   343     return nullptr;

   344   }

   346   nsRefPtr<Image> image;

   347   switch(aColorFormat) {

   348     case MPAPI::RGB565:

   349       image = mozilla::layers::CreateSharedRGBImage(mImageContainer,

   350                                                     nsIntSize(aWidth, aHeight),

   351                                                     gfxImageFormat::RGB16_565);

   352       if (!image) {

   353         NS_WARNING("Could not create rgb image");

   354         return nullptr;

   355       }

   357       mImage = image;

   358       return image->AsSharedImage()->GetBuffer();

   359     case MPAPI::I420:

   360       return CreateI420Image(aWidth, aHeight);

   361     default:

   362       NS_NOTREACHED("Color format not supported");

   363       return nullptr;

   364   }

   365 }

   367 uint8_t *

   368 MediaPluginReader::ImageBufferCallback::CreateI420Image(size_t aWidth,

   369                                                         size_t aHeight)

   370 {

   371   mImage = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR);

   372   PlanarYCbCrImage *yuvImage = static_cast<PlanarYCbCrImage *>(mImage.get());

   374   if (!yuvImage) {

   375     NS_WARNING("Could not create I420 image");

   376     return nullptr;

   377   }

   379   size_t frameSize = aWidth * aHeight;

   381   // Allocate enough for one full resolution Y plane

   382   // and two quarter resolution Cb/Cr planes.

   383   uint8_t *buffer = yuvImage->AllocateAndGetNewBuffer(frameSize * 3 / 2);

   385   mozilla::layers::PlanarYCbCrData frameDesc;

   387   frameDesc.mYChannel = buffer;

   388   frameDesc.mCbChannel = buffer + frameSize;

   389   frameDesc.mCrChannel = buffer + frameSize * 5 / 4;

   391   frameDesc.mYSize = IntSize(aWidth, aHeight);

   392   frameDesc.mCbCrSize = IntSize(aWidth / 2, aHeight / 2);

   394   frameDesc.mYStride = aWidth;

   395   frameDesc.mCbCrStride = aWidth / 2;

   397   frameDesc.mYSkip = 0;

   398   frameDesc.mCbSkip = 0;

   399   frameDesc.mCrSkip = 0;

   401   frameDesc.mPicX = 0;

   402   frameDesc.mPicY = 0;

   403   frameDesc.mPicSize = IntSize(aWidth, aHeight);

   405   yuvImage->SetDataNoCopy(frameDesc);

   407   return buffer;

   408 }

   410 already_AddRefed<Image>

   411 MediaPluginReader::ImageBufferCallback::GetImage()

   412 {

   413   return mImage.forget();

   414 }

   416 } // namespace mozilla

The Tor Browser / file revision

content/media/plugins/MediaPluginReader.cpp@97036ab72558

content/media/plugins/MediaPluginReader.cpp