michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "VP8TrackEncoder.h"
michael@0: #include "vpx/vp8cx.h"
michael@0: #include "vpx/vpx_encoder.h"
michael@0: #include "VideoUtils.h"
michael@0: #include "prsystem.h"
michael@0: #include "WebMWriter.h"
michael@0: #include "libyuv.h"
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: #ifdef PR_LOGGING
michael@0: PRLogModuleInfo* gVP8TrackEncoderLog;
michael@0: #define VP8LOG(msg, ...) PR_LOG(gVP8TrackEncoderLog, PR_LOG_DEBUG, \
michael@0:                                   (msg, ##__VA_ARGS__))
michael@0: // Debug logging macro with object pointer and class name.
michael@0: #else
michael@0: #define VP8LOG(msg, ...)
michael@0: #endif
michael@0: 
michael@0: #define DEFAULT_BITRATE 2500 // in kbit/s
michael@0: #define DEFAULT_ENCODE_FRAMERATE 30
michael@0: 
michael@0: using namespace mozilla::layers;
michael@0: 
michael@0: VP8TrackEncoder::VP8TrackEncoder()
michael@0:   : VideoTrackEncoder()
michael@0:   , mEncodedFrameDuration(0)
michael@0:   , mEncodedTimestamp(0)
michael@0:   , mRemainingTicks(0)
michael@0:   , mVPXContext(new vpx_codec_ctx_t())
michael@0:   , mVPXImageWrapper(new vpx_image_t())
michael@0: {
michael@0:   MOZ_COUNT_CTOR(VP8TrackEncoder);
michael@0: #ifdef PR_LOGGING
michael@0:   if (!gVP8TrackEncoderLog) {
michael@0:     gVP8TrackEncoderLog = PR_NewLogModule("VP8TrackEncoder");
michael@0:   }
michael@0: #endif
michael@0: }
michael@0: 
michael@0: VP8TrackEncoder::~VP8TrackEncoder()
michael@0: {
michael@0:   if (mInitialized) {
michael@0:     vpx_codec_destroy(mVPXContext);
michael@0:   }
michael@0: 
michael@0:   if (mVPXImageWrapper) {
michael@0:     vpx_img_free(mVPXImageWrapper);
michael@0:   }
michael@0:   MOZ_COUNT_DTOR(VP8TrackEncoder);
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: VP8TrackEncoder::Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
michael@0:                       int32_t aDisplayHeight,TrackRate aTrackRate)
michael@0: {
michael@0:   if (aWidth < 1 || aHeight < 1 || aDisplayWidth < 1 || aDisplayHeight < 1
michael@0:       || aTrackRate <= 0) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
michael@0: 
michael@0:   mTrackRate = aTrackRate;
michael@0:   mEncodedFrameRate = DEFAULT_ENCODE_FRAMERATE;
michael@0:   mEncodedFrameDuration = mTrackRate / mEncodedFrameRate;
michael@0:   mFrameWidth = aWidth;
michael@0:   mFrameHeight = aHeight;
michael@0:   mDisplayWidth = aDisplayWidth;
michael@0:   mDisplayHeight = aDisplayHeight;
michael@0: 
michael@0:   // Encoder configuration structure.
michael@0:   vpx_codec_enc_cfg_t config;
michael@0:   memset(&config, 0, sizeof(vpx_codec_enc_cfg_t));
michael@0:   if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config, 0)) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   // Creating a wrapper to the image - setting image data to NULL. Actual
michael@0:   // pointer will be set in encode. Setting align to 1, as it is meaningless
michael@0:   // (actual memory is not allocated).
michael@0:   vpx_img_wrap(mVPXImageWrapper, IMG_FMT_I420,
michael@0:                mFrameWidth, mFrameHeight, 1, nullptr);
michael@0: 
michael@0:   config.g_w = mFrameWidth;
michael@0:   config.g_h = mFrameHeight;
michael@0:   // TODO: Maybe we should have various aFrameRate bitrate pair for each devices?
michael@0:   // or for different platform
michael@0:   config.rc_target_bitrate = DEFAULT_BITRATE; // in kbit/s
michael@0: 
michael@0:   // Setting the time base of the codec
michael@0:   config.g_timebase.num = 1;
michael@0:   config.g_timebase.den = mTrackRate;
michael@0: 
michael@0:   config.g_error_resilient = 0;
michael@0: 
michael@0:   config.g_lag_in_frames = 0; // 0- no frame lagging
michael@0: 
michael@0:   int32_t number_of_cores = PR_GetNumberOfProcessors();
michael@0:   if (mFrameWidth * mFrameHeight > 1280 * 960 && number_of_cores >= 6) {
michael@0:     config.g_threads = 3; // 3 threads for 1080p.
michael@0:   } else if (mFrameWidth * mFrameHeight > 640 * 480 && number_of_cores >= 3) {
michael@0:     config.g_threads = 2; // 2 threads for qHD/HD.
michael@0:   } else {
michael@0:     config.g_threads = 1; // 1 thread for VGA or less
michael@0:   }
michael@0: 
michael@0:   // rate control settings
michael@0:   config.rc_dropframe_thresh = 0;
michael@0:   config.rc_end_usage = VPX_CBR;
michael@0:   config.g_pass = VPX_RC_ONE_PASS;
michael@0:   config.rc_resize_allowed = 1;
michael@0:   config.rc_undershoot_pct = 100;
michael@0:   config.rc_overshoot_pct = 15;
michael@0:   config.rc_buf_initial_sz = 500;
michael@0:   config.rc_buf_optimal_sz = 600;
michael@0:   config.rc_buf_sz = 1000;
michael@0: 
michael@0:   config.kf_mode = VPX_KF_AUTO;
michael@0:   // Ensure that we can output one I-frame per second.
michael@0:   config.kf_max_dist = mEncodedFrameRate;
michael@0: 
michael@0:   vpx_codec_flags_t flags = 0;
michael@0:   flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
michael@0:   if (vpx_codec_enc_init(mVPXContext, vpx_codec_vp8_cx(), &config, flags)) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   vpx_codec_control(mVPXContext, VP8E_SET_STATIC_THRESHOLD, 1);
michael@0:   vpx_codec_control(mVPXContext, VP8E_SET_CPUUSED, -6);
michael@0:   vpx_codec_control(mVPXContext, VP8E_SET_TOKEN_PARTITIONS,
michael@0:                     VP8_ONE_TOKENPARTITION);
michael@0: 
michael@0:   mInitialized = true;
michael@0:   mon.NotifyAll();
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: already_AddRefed<TrackMetadataBase>
michael@0: VP8TrackEncoder::GetMetadata()
michael@0: {
michael@0:   {
michael@0:     // Wait if mEncoder is not initialized.
michael@0:     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
michael@0:     while (!mCanceled && !mInitialized) {
michael@0:       mon.Wait();
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   if (mCanceled || mEncodingComplete) {
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   nsRefPtr<VP8Metadata> meta = new VP8Metadata();
michael@0:   meta->mWidth = mFrameWidth;
michael@0:   meta->mHeight = mFrameHeight;
michael@0:   meta->mDisplayWidth = mDisplayWidth;
michael@0:   meta->mDisplayHeight = mDisplayHeight;
michael@0:   meta->mEncodedFrameRate = mEncodedFrameRate;
michael@0: 
michael@0:   return meta.forget();
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: VP8TrackEncoder::GetEncodedPartitions(EncodedFrameContainer& aData)
michael@0: {
michael@0:   vpx_codec_iter_t iter = nullptr;
michael@0:   EncodedFrame::FrameType frameType = EncodedFrame::VP8_P_FRAME;
michael@0:   nsTArray<uint8_t> frameData;
michael@0:   nsresult rv;
michael@0:   const vpx_codec_cx_pkt_t *pkt = nullptr;
michael@0:   while ((pkt = vpx_codec_get_cx_data(mVPXContext, &iter)) != nullptr) {
michael@0:     switch (pkt->kind) {
michael@0:       case VPX_CODEC_CX_FRAME_PKT: {
michael@0:         // Copy the encoded data from libvpx to frameData
michael@0:         frameData.AppendElements((uint8_t*)pkt->data.frame.buf,
michael@0:                                  pkt->data.frame.sz);
michael@0:         break;
michael@0:       }
michael@0:       default: {
michael@0:         break;
michael@0:       }
michael@0:     }
michael@0:     // End of frame
michael@0:     if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) {
michael@0:       if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
michael@0:         frameType = EncodedFrame::VP8_I_FRAME;
michael@0:       }
michael@0:       break;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   if (!frameData.IsEmpty() &&
michael@0:       (pkt->data.frame.pts == mEncodedTimestamp)) {
michael@0:     // Copy the encoded data to aData.
michael@0:     EncodedFrame* videoData = new EncodedFrame();
michael@0:     videoData->SetFrameType(frameType);
michael@0:     // Convert the timestamp and duration to Usecs.
michael@0:     CheckedInt64 timestamp = FramesToUsecs(mEncodedTimestamp, mTrackRate);
michael@0:     if (timestamp.isValid()) {
michael@0:       videoData->SetTimeStamp(
michael@0:         (uint64_t)FramesToUsecs(mEncodedTimestamp, mTrackRate).value());
michael@0:     }
michael@0:     CheckedInt64 duration = FramesToUsecs(pkt->data.frame.duration, mTrackRate);
michael@0:     if (duration.isValid()) {
michael@0:       videoData->SetDuration(
michael@0:         (uint64_t)FramesToUsecs(pkt->data.frame.duration, mTrackRate).value());
michael@0:     }
michael@0:     rv = videoData->SwapInFrameData(frameData);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:     VP8LOG("GetEncodedPartitions TimeStamp %lld Duration %lld\n",
michael@0:            videoData->GetTimeStamp(), videoData->GetDuration());
michael@0:     VP8LOG("frameType %d\n", videoData->GetFrameType());
michael@0:     aData.AppendEncodedFrame(videoData);
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: void VP8TrackEncoder::PrepareMutedFrame()
michael@0: {
michael@0:   if (mMuteFrame.IsEmpty()) {
michael@0:     CreateMutedFrame(&mMuteFrame);
michael@0:   }
michael@0: 
michael@0:   uint32_t yPlaneSize = mFrameWidth * mFrameHeight;
michael@0:   uint32_t halfWidth = (mFrameWidth + 1) / 2;
michael@0:   uint32_t halfHeight = (mFrameHeight + 1) / 2;
michael@0:   uint32_t uvPlaneSize = halfWidth * halfHeight;
michael@0: 
michael@0:   MOZ_ASSERT(mMuteFrame.Length() >= (yPlaneSize + uvPlaneSize * 2));
michael@0:   uint8_t *y = mMuteFrame.Elements();
michael@0:   uint8_t *cb = mMuteFrame.Elements() + yPlaneSize;
michael@0:   uint8_t *cr = mMuteFrame.Elements() + yPlaneSize + uvPlaneSize;
michael@0: 
michael@0:   mVPXImageWrapper->planes[PLANE_Y] = y;
michael@0:   mVPXImageWrapper->planes[PLANE_U] = cb;
michael@0:   mVPXImageWrapper->planes[PLANE_V] = cr;
michael@0:   mVPXImageWrapper->stride[VPX_PLANE_Y] = mFrameWidth;
michael@0:   mVPXImageWrapper->stride[VPX_PLANE_U] = halfWidth;
michael@0:   mVPXImageWrapper->stride[VPX_PLANE_V] = halfWidth;
michael@0: }
michael@0: 
michael@0: static bool isYUV420(const PlanarYCbCrImage::Data *aData)
michael@0: {
michael@0:   if (aData->mYSize == aData->mCbCrSize * 2) {
michael@0:     return true;
michael@0:   }
michael@0:   return false;
michael@0: }
michael@0: 
michael@0: static bool isYUV422(const PlanarYCbCrImage::Data *aData)
michael@0: {
michael@0:   if ((aData->mYSize.width == aData->mCbCrSize.width * 2) &&
michael@0:       (aData->mYSize.height == aData->mCbCrSize.height)) {
michael@0:     return true;
michael@0:   }
michael@0:   return false;
michael@0: }
michael@0: 
michael@0: static bool isYUV444(const PlanarYCbCrImage::Data *aData)
michael@0: {
michael@0:   if (aData->mYSize == aData->mCbCrSize) {
michael@0:     return true;
michael@0:   }
michael@0:   return false;
michael@0: }
michael@0: 
michael@0: nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk &aChunk)
michael@0: {
michael@0:   if (aChunk.mFrame.GetForceBlack() || aChunk.IsNull()) {
michael@0:     PrepareMutedFrame();
michael@0:   } else {
michael@0:     Image* img = aChunk.mFrame.GetImage();
michael@0:     ImageFormat format = img->GetFormat();
michael@0:     if (format != ImageFormat::PLANAR_YCBCR) {
michael@0:       VP8LOG("Unsupported video format\n");
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0: 
michael@0:     // Cast away constness b/c some of the accessors are non-const
michael@0:     PlanarYCbCrImage* yuv =
michael@0:     const_cast<PlanarYCbCrImage *>(static_cast<const PlanarYCbCrImage *>(img));
michael@0:     // Big-time assumption here that this is all contiguous data coming
michael@0:     // from getUserMedia or other sources.
michael@0:     MOZ_ASSERT(yuv);
michael@0:     if (!yuv->IsValid()) {
michael@0:       NS_WARNING("PlanarYCbCrImage is not valid");
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0:     const PlanarYCbCrImage::Data *data = yuv->GetData();
michael@0: 
michael@0:     if (isYUV420(data) && !data->mCbSkip) { // 420 planar
michael@0:       mVPXImageWrapper->planes[PLANE_Y] = data->mYChannel;
michael@0:       mVPXImageWrapper->planes[PLANE_U] = data->mCbChannel;
michael@0:       mVPXImageWrapper->planes[PLANE_V] = data->mCrChannel;
michael@0:       mVPXImageWrapper->stride[VPX_PLANE_Y] = data->mYStride;
michael@0:       mVPXImageWrapper->stride[VPX_PLANE_U] = data->mCbCrStride;
michael@0:       mVPXImageWrapper->stride[VPX_PLANE_V] = data->mCbCrStride;
michael@0:     } else {
michael@0:       uint32_t yPlaneSize = mFrameWidth * mFrameHeight;
michael@0:       uint32_t halfWidth = (mFrameWidth + 1) / 2;
michael@0:       uint32_t halfHeight = (mFrameHeight + 1) / 2;
michael@0:       uint32_t uvPlaneSize = halfWidth * halfHeight;
michael@0:       if (mI420Frame.IsEmpty()) {
michael@0:         mI420Frame.SetLength(yPlaneSize + uvPlaneSize * 2);
michael@0:       }
michael@0: 
michael@0:       MOZ_ASSERT(mI420Frame.Length() >= (yPlaneSize + uvPlaneSize * 2));
michael@0:       uint8_t *y = mI420Frame.Elements();
michael@0:       uint8_t *cb = mI420Frame.Elements() + yPlaneSize;
michael@0:       uint8_t *cr = mI420Frame.Elements() + yPlaneSize + uvPlaneSize;
michael@0: 
michael@0:       if (isYUV420(data) && data->mCbSkip) {
michael@0:         // If mCbSkip is set, we assume it's nv12 or nv21.
michael@0:         if (data->mCbChannel < data->mCrChannel) { // nv12
michael@0:           libyuv::NV12ToI420(data->mYChannel, data->mYStride,
michael@0:                              data->mCbChannel, data->mCbCrStride,
michael@0:                              y, mFrameWidth,
michael@0:                              cb, halfWidth,
michael@0:                              cr, halfWidth,
michael@0:                              mFrameWidth, mFrameHeight);
michael@0:         } else { // nv21
michael@0:           libyuv::NV21ToI420(data->mYChannel, data->mYStride,
michael@0:                              data->mCrChannel, data->mCbCrStride,
michael@0:                              y, mFrameWidth,
michael@0:                              cb, halfWidth,
michael@0:                              cr, halfWidth,
michael@0:                              mFrameWidth, mFrameHeight);
michael@0:         }
michael@0:       } else if (isYUV444(data) && !data->mCbSkip) {
michael@0:         libyuv::I444ToI420(data->mYChannel, data->mYStride,
michael@0:                            data->mCbChannel, data->mCbCrStride,
michael@0:                            data->mCrChannel, data->mCbCrStride,
michael@0:                            y, mFrameWidth,
michael@0:                            cb, halfWidth,
michael@0:                            cr, halfWidth,
michael@0:                            mFrameWidth, mFrameHeight);
michael@0:       } else if (isYUV422(data) && !data->mCbSkip) {
michael@0:         libyuv::I422ToI420(data->mYChannel, data->mYStride,
michael@0:                            data->mCbChannel, data->mCbCrStride,
michael@0:                            data->mCrChannel, data->mCbCrStride,
michael@0:                            y, mFrameWidth,
michael@0:                            cb, halfWidth,
michael@0:                            cr, halfWidth,
michael@0:                            mFrameWidth, mFrameHeight);
michael@0:       } else {
michael@0:         VP8LOG("Unsupported planar format\n");
michael@0:         return NS_ERROR_NOT_IMPLEMENTED;
michael@0:       }
michael@0: 
michael@0:       mVPXImageWrapper->planes[PLANE_Y] = y;
michael@0:       mVPXImageWrapper->planes[PLANE_U] = cb;
michael@0:       mVPXImageWrapper->planes[PLANE_V] = cr;
michael@0:       mVPXImageWrapper->stride[VPX_PLANE_Y] = mFrameWidth;
michael@0:       mVPXImageWrapper->stride[VPX_PLANE_U] = halfWidth;
michael@0:       mVPXImageWrapper->stride[VPX_PLANE_V] = halfWidth;
michael@0:     }
michael@0:   }
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: // These two define value used in GetNextEncodeOperation to determine the
michael@0: // EncodeOperation for next target frame.
michael@0: #define I_FRAME_RATIO (0.5)
michael@0: #define SKIP_FRAME_RATIO (0.75)
michael@0: 
michael@0: /**
michael@0:  * Compares the elapsed time from the beginning of GetEncodedTrack and
michael@0:  * the processed frame duration in mSourceSegment
michael@0:  * in order to set the nextEncodeOperation for next target frame.
michael@0:  */
michael@0: VP8TrackEncoder::EncodeOperation
michael@0: VP8TrackEncoder::GetNextEncodeOperation(TimeDuration aTimeElapsed,
michael@0:                                         TrackTicks aProcessedDuration)
michael@0: {
michael@0:   int64_t durationInUsec =
michael@0:     FramesToUsecs(aProcessedDuration + mEncodedFrameDuration,
michael@0:                   mTrackRate).value();
michael@0:   if (aTimeElapsed.ToMicroseconds() > (durationInUsec * SKIP_FRAME_RATIO)) {
michael@0:     // The encoder is too slow.
michael@0:     // We should skip next frame to consume the mSourceSegment.
michael@0:     return SKIP_FRAME;
michael@0:   } else if (aTimeElapsed.ToMicroseconds() > (durationInUsec * I_FRAME_RATIO)) {
michael@0:     // The encoder is a little slow.
michael@0:     // We force the encoder to encode an I-frame to accelerate.
michael@0:     return ENCODE_I_FRAME;
michael@0:   } else {
michael@0:     return ENCODE_NORMAL_FRAME;
michael@0:   }
michael@0: }
michael@0: 
michael@0: TrackTicks
michael@0: VP8TrackEncoder::CalculateRemainingTicks(TrackTicks aDurationCopied,
michael@0:                                          TrackTicks aEncodedDuration)
michael@0: {
michael@0:   return mRemainingTicks + aEncodedDuration - aDurationCopied;
michael@0: }
michael@0: 
michael@0: // Try to extend the encodedDuration as long as possible if the target frame
michael@0: // has a long duration.
michael@0: TrackTicks
michael@0: VP8TrackEncoder::CalculateEncodedDuration(TrackTicks aDurationCopied)
michael@0: {
michael@0:   TrackTicks temp64 = aDurationCopied;
michael@0:   TrackTicks encodedDuration = mEncodedFrameDuration;
michael@0:   temp64 -= mRemainingTicks;
michael@0:   while (temp64 > mEncodedFrameDuration) {
michael@0:     temp64 -= mEncodedFrameDuration;
michael@0:     encodedDuration += mEncodedFrameDuration;
michael@0:   }
michael@0:   return encodedDuration;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Encoding flow in GetEncodedTrack():
michael@0:  * 1: Check the mInitialized state and the packet duration.
michael@0:  * 2: Move the data from mRawSegment to mSourceSegment.
michael@0:  * 3: Encode the video chunks in mSourceSegment in a for-loop.
michael@0:  * 3.1: Pick the video chunk by mRemainingTicks.
michael@0:  * 3.2: Calculate the encoding duration for the parameter of vpx_codec_encode().
michael@0:  *      The encoding duration is a multiple of mEncodedFrameDuration.
michael@0:  * 3.3: Setup the video chunk to mVPXImageWrapper by PrepareRawFrame().
michael@0:  * 3.4: Send frame into vp8 encoder by vpx_codec_encode().
michael@0:  * 3.5: Get the output frame from encoder by calling GetEncodedPartitions().
michael@0:  * 3.6: Calculate the mRemainingTicks for next target frame.
michael@0:  * 3.7: Set the nextEncodeOperation for the next target frame.
michael@0:  *      There is a heuristic: If the frame duration we have processed in
michael@0:  *      mSourceSegment is 100ms, means that we can't spend more than 100ms to
michael@0:  *      encode it.
michael@0:  * 4. Remove the encoded chunks in mSourceSegment after for-loop.
michael@0:  *
michael@0:  * Ex1: Input frame rate is 100 => input frame duration is 10ms for each.
michael@0:  *     mEncodedFrameRate is 30 => output frame duration is 33ms.
michael@0:  *     In this case, the frame duration in mSourceSegment will be:
michael@0:  *     1st : 0~10ms
michael@0:  *     2nd : 10~20ms
michael@0:  *     3rd : 20~30ms
michael@0:  *     4th : 30~40ms
michael@0:  *     ...
michael@0:  *     The VP8 encoder will take the 1st and 4th frames to encode. At beginning
michael@0:  *     mRemainingTicks is 0 for 1st frame, then the mRemainingTicks is set
michael@0:  *     to 23 to pick the 4th frame. (mEncodedFrameDuration - 1st frame duration)
michael@0:  *
michael@0:  * Ex2: Input frame rate is 25 => frame duration is 40ms for each.
michael@0:  *     mEncodedFrameRate is 30 => output frame duration is 33ms.
michael@0:  *     In this case, the frame duration in mSourceSegment will be:
michael@0:  *     1st : 0~40ms
michael@0:  *     2nd : 40~80ms
michael@0:  *     3rd : 80~120ms
michael@0:  *     4th : 120~160ms
michael@0:  *     ...
michael@0:  *     Because the input frame duration is 40ms larger than 33ms, so the first
michael@0:  *     encoded frame duration will be 66ms by calling CalculateEncodedDuration.
michael@0:  *     And the mRemainingTicks will be set to 26
michael@0:  *     (CalculateRemainingTicks 0+66-40) in order to pick the next frame(2nd)
michael@0:  *     in mSourceSegment.
michael@0:  */
michael@0: nsresult
michael@0: VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
michael@0: {
michael@0:   {
michael@0:     // Move all the samples from mRawSegment to mSourceSegment. We only hold
michael@0:     // the monitor in this block.
michael@0:     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
michael@0:     // Wait if mEncoder is not initialized, or when not enough raw data, but is
michael@0:     // not the end of stream nor is being canceled.
michael@0:     while (!mCanceled && (!mInitialized ||
michael@0:            (mRawSegment.GetDuration() + mSourceSegment.GetDuration() <
michael@0:             mEncodedFrameDuration && !mEndOfStream))) {
michael@0:       mon.Wait();
michael@0:     }
michael@0:     if (mCanceled || mEncodingComplete) {
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0:     mSourceSegment.AppendFrom(&mRawSegment);
michael@0:   }
michael@0: 
michael@0:   VideoSegment::ChunkIterator iter(mSourceSegment);
michael@0:   TrackTicks durationCopied = 0;
michael@0:   TrackTicks totalProcessedDuration = 0;
michael@0:   TimeStamp timebase = TimeStamp::Now();
michael@0:   EncodeOperation nextEncodeOperation = ENCODE_NORMAL_FRAME;
michael@0: 
michael@0:   for (; !iter.IsEnded(); iter.Next()) {
michael@0:     VideoChunk &chunk = *iter;
michael@0:     // Accumulate chunk's duration to durationCopied until it reaches
michael@0:     // mRemainingTicks.
michael@0:     durationCopied += chunk.GetDuration();
michael@0:     MOZ_ASSERT(mRemainingTicks <= mEncodedFrameDuration);
michael@0:     VP8LOG("durationCopied %lld mRemainingTicks %lld\n",
michael@0:            durationCopied, mRemainingTicks);
michael@0:     if (durationCopied >= mRemainingTicks) {
michael@0:       VP8LOG("nextEncodeOperation is %d\n",nextEncodeOperation);
michael@0:       // Calculate encodedDuration for this target frame.
michael@0:       TrackTicks encodedDuration = CalculateEncodedDuration(durationCopied);
michael@0: 
michael@0:       // Encode frame.
michael@0:       if (nextEncodeOperation != SKIP_FRAME) {
michael@0:         nsresult rv = PrepareRawFrame(chunk);
michael@0:         NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
michael@0: 
michael@0:         // Encode the data with VP8 encoder
michael@0:         int flags = (nextEncodeOperation == ENCODE_NORMAL_FRAME) ?
michael@0:                     0 : VPX_EFLAG_FORCE_KF;
michael@0:         if (vpx_codec_encode(mVPXContext, mVPXImageWrapper, mEncodedTimestamp,
michael@0:                              (unsigned long)encodedDuration, flags,
michael@0:                              VPX_DL_REALTIME)) {
michael@0:           return NS_ERROR_FAILURE;
michael@0:         }
michael@0:         // Get the encoded data from VP8 encoder.
michael@0:         GetEncodedPartitions(aData);
michael@0:       } else {
michael@0:         // SKIP_FRAME
michael@0:         // Extend the duration of the last encoded data in aData
michael@0:         // because this frame will be skip.
michael@0:         nsRefPtr<EncodedFrame> last = nullptr;
michael@0:         last = aData.GetEncodedFrames().LastElement();
michael@0:         if (last) {
michael@0:           last->SetDuration(last->GetDuration() + encodedDuration);
michael@0:         }
michael@0:       }
michael@0:       // Move forward the mEncodedTimestamp.
michael@0:       mEncodedTimestamp += encodedDuration;
michael@0:       totalProcessedDuration += durationCopied;
michael@0:       // Calculate mRemainingTicks for next target frame.
michael@0:       mRemainingTicks = CalculateRemainingTicks(durationCopied,
michael@0:                                                 encodedDuration);
michael@0: 
michael@0:       // Check the remain data is enough for next target frame.
michael@0:       if (mSourceSegment.GetDuration() - totalProcessedDuration
michael@0:           >= mEncodedFrameDuration) {
michael@0:         TimeDuration elapsedTime = TimeStamp::Now() - timebase;
michael@0:         nextEncodeOperation = GetNextEncodeOperation(elapsedTime,
michael@0:                                                      totalProcessedDuration);
michael@0:         // Reset durationCopied for next iteration.
michael@0:         durationCopied = 0;
michael@0:       } else {
michael@0:         // Process done, there is no enough data left for next iteration,
michael@0:         // break the for-loop.
michael@0:         break;
michael@0:       }
michael@0:     }
michael@0:   }
michael@0:   // Remove the chunks we have processed.
michael@0:   mSourceSegment.RemoveLeading(totalProcessedDuration);
michael@0:   VP8LOG("RemoveLeading %lld\n",totalProcessedDuration);
michael@0: 
michael@0:   // End of stream, pull the rest frames in encoder.
michael@0:   if (mEndOfStream) {
michael@0:     VP8LOG("mEndOfStream is true\n");
michael@0:     mEncodingComplete = true;
michael@0:     if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp,
michael@0:                          mEncodedFrameDuration, 0, VPX_DL_REALTIME)) {
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0:     GetEncodedPartitions(aData);
michael@0:   }
michael@0: 
michael@0:   return NS_OK ;
michael@0: }
michael@0: 
michael@0: } // namespace mozilla