michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: #include "TrackEncoder.h"
michael@0: #include "AudioChannelFormat.h"
michael@0: #include "MediaStreamGraph.h"
michael@0: #include "prlog.h"
michael@0: #include "VideoUtils.h"
michael@0: 
michael@0: #undef LOG
michael@0: #ifdef MOZ_WIDGET_GONK
michael@0: #include <android/log.h>
michael@0: #define LOG(args...) __android_log_print(ANDROID_LOG_INFO, "MediaEncoder", ## args);
michael@0: #else
michael@0: #define LOG(args, ...)
michael@0: #endif
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: #ifdef PR_LOGGING
michael@0: PRLogModuleInfo* gTrackEncoderLog;
michael@0: #define TRACK_LOG(type, msg) PR_LOG(gTrackEncoderLog, type, msg)
michael@0: #else
michael@0: #define TRACK_LOG(type, msg)
michael@0: #endif
michael@0: 
michael@0: static const int DEFAULT_CHANNELS = 1;
michael@0: static const int DEFAULT_SAMPLING_RATE = 16000;
michael@0: static const int DEFAULT_FRAME_WIDTH = 640;
michael@0: static const int DEFAULT_FRAME_HEIGHT = 480;
michael@0: static const int DEFAULT_TRACK_RATE = USECS_PER_S;
michael@0: 
michael@0: TrackEncoder::TrackEncoder()
michael@0:   : mReentrantMonitor("media.TrackEncoder")
michael@0:   , mEncodingComplete(false)
michael@0:   , mEosSetInEncoder(false)
michael@0:   , mInitialized(false)
michael@0:   , mEndOfStream(false)
michael@0:   , mCanceled(false)
michael@0: #ifdef PR_LOGGING
michael@0:   , mAudioInitCounter(0)
michael@0:   , mVideoInitCounter(0)
michael@0: #endif
michael@0: {
michael@0: #ifdef PR_LOGGING
michael@0:   if (!gTrackEncoderLog) {
michael@0:     gTrackEncoderLog = PR_NewLogModule("TrackEncoder");
michael@0:   }
michael@0: #endif
michael@0: }
michael@0: 
michael@0: void
michael@0: AudioTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
michael@0:                                             TrackID aID,
michael@0:                                             TrackRate aTrackRate,
michael@0:                                             TrackTicks aTrackOffset,
michael@0:                                             uint32_t aTrackEvents,
michael@0:                                             const MediaSegment& aQueuedMedia)
michael@0: {
michael@0:   if (mCanceled) {
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   const AudioSegment& audio = static_cast<const AudioSegment&>(aQueuedMedia);
michael@0: 
michael@0:   // Check and initialize parameters for codec encoder.
michael@0:   if (!mInitialized) {
michael@0: #ifdef PR_LOGGING
michael@0:     mAudioInitCounter++;
michael@0:     TRACK_LOG(PR_LOG_DEBUG, ("Init the audio encoder %d times", mAudioInitCounter));
michael@0: #endif
michael@0:     AudioSegment::ChunkIterator iter(const_cast<AudioSegment&>(audio));
michael@0:     while (!iter.IsEnded()) {
michael@0:       AudioChunk chunk = *iter;
michael@0: 
michael@0:       // The number of channels is determined by the first non-null chunk, and
michael@0:       // thus the audio encoder is initialized at this time.
michael@0:       if (!chunk.IsNull()) {
michael@0:         nsresult rv = Init(chunk.mChannelData.Length(), aTrackRate);
michael@0:         if (NS_FAILED(rv)) {
michael@0:           LOG("[AudioTrackEncoder]: Fail to initialize the encoder!");
michael@0:           NotifyCancel();
michael@0:         }
michael@0:         break;
michael@0:       }
michael@0: 
michael@0:       iter.Next();
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Append and consume this raw segment.
michael@0:   AppendAudioSegment(audio);
michael@0: 
michael@0: 
michael@0:   // The stream has stopped and reached the end of track.
michael@0:   if (aTrackEvents == MediaStreamListener::TRACK_EVENT_ENDED) {
michael@0:     LOG("[AudioTrackEncoder]: Receive TRACK_EVENT_ENDED .");
michael@0:     NotifyEndOfStream();
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: AudioTrackEncoder::NotifyEndOfStream()
michael@0: {
michael@0:   // If source audio track is completely silent till the end of encoding,
michael@0:   // initialize the encoder with default channel counts and sampling rate.
michael@0:   if (!mCanceled && !mInitialized) {
michael@0:     Init(DEFAULT_CHANNELS, DEFAULT_SAMPLING_RATE);
michael@0:   }
michael@0: 
michael@0:   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
michael@0:   mEndOfStream = true;
michael@0:   mReentrantMonitor.NotifyAll();
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: AudioTrackEncoder::AppendAudioSegment(const AudioSegment& aSegment)
michael@0: {
michael@0:   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
michael@0: 
michael@0:   AudioSegment::ChunkIterator iter(const_cast<AudioSegment&>(aSegment));
michael@0:   while (!iter.IsEnded()) {
michael@0:     AudioChunk chunk = *iter;
michael@0:     // Append and consume both non-null and null chunks.
michael@0:     mRawSegment.AppendAndConsumeChunk(&chunk);
michael@0:     iter.Next();
michael@0:   }
michael@0: 
michael@0:   if (mRawSegment.GetDuration() >= GetPacketDuration()) {
michael@0:     mReentrantMonitor.NotifyAll();
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: static const int AUDIO_PROCESSING_FRAMES = 640; /* > 10ms of 48KHz audio */
michael@0: static const uint8_t gZeroChannel[MAX_AUDIO_SAMPLE_SIZE*AUDIO_PROCESSING_FRAMES] = {0};
michael@0: 
michael@0: /*static*/
michael@0: void
michael@0: AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
michael@0:                                        int32_t aDuration,
michael@0:                                        uint32_t aOutputChannels,
michael@0:                                        AudioDataValue* aOutput)
michael@0: {
michael@0:   if (aChunk.mChannelData.Length() < aOutputChannels) {
michael@0:     // Up-mix. This might make the mChannelData have more than aChannels.
michael@0:     AudioChannelsUpMix(&aChunk.mChannelData, aOutputChannels, gZeroChannel);
michael@0:   }
michael@0: 
michael@0:   if (aChunk.mChannelData.Length() > aOutputChannels) {
michael@0:     DownmixAndInterleave(aChunk.mChannelData, aChunk.mBufferFormat, aDuration,
michael@0:                          aChunk.mVolume, aOutputChannels, aOutput);
michael@0:   } else {
michael@0:     InterleaveAndConvertBuffer(aChunk.mChannelData.Elements(),
michael@0:                                aChunk.mBufferFormat, aDuration, aChunk.mVolume,
michael@0:                                aOutputChannels, aOutput);
michael@0:   }
michael@0: }
michael@0: 
michael@0: /*static*/
michael@0: void
michael@0: AudioTrackEncoder::DeInterleaveTrackData(AudioDataValue* aInput,
michael@0:                                          int32_t aDuration,
michael@0:                                          int32_t aChannels,
michael@0:                                          AudioDataValue* aOutput)
michael@0: {
michael@0:   for (int32_t i = 0; i < aChannels; ++i) {
michael@0:     for(int32_t j = 0; j < aDuration; ++j) {
michael@0:       aOutput[i * aDuration + j] = aInput[i + j * aChannels];
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: VideoTrackEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
michael@0:                                             TrackID aID,
michael@0:                                             TrackRate aTrackRate,
michael@0:                                             TrackTicks aTrackOffset,
michael@0:                                             uint32_t aTrackEvents,
michael@0:                                             const MediaSegment& aQueuedMedia)
michael@0: {
michael@0:   if (mCanceled) {
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
michael@0: 
michael@0:    // Check and initialize parameters for codec encoder.
michael@0:   if (!mInitialized) {
michael@0: #ifdef PR_LOGGING
michael@0:     mVideoInitCounter++;
michael@0:     TRACK_LOG(PR_LOG_DEBUG, ("Init the video encoder %d times", mVideoInitCounter));
michael@0: #endif
michael@0:     VideoSegment::ChunkIterator iter(const_cast<VideoSegment&>(video));
michael@0:     while (!iter.IsEnded()) {
michael@0:       VideoChunk chunk = *iter;
michael@0:       if (!chunk.IsNull()) {
michael@0:         gfx::IntSize imgsize = chunk.mFrame.GetImage()->GetSize();
michael@0:         gfxIntSize intrinsicSize = chunk.mFrame.GetIntrinsicSize();
michael@0:         nsresult rv = Init(imgsize.width, imgsize.height,
michael@0:                            intrinsicSize.width, intrinsicSize.height,
michael@0:                            aTrackRate);
michael@0:         if (NS_FAILED(rv)) {
michael@0:           LOG("[VideoTrackEncoder]: Fail to initialize the encoder!");
michael@0:           NotifyCancel();
michael@0:         }
michael@0:         break;
michael@0:       }
michael@0: 
michael@0:       iter.Next();
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   AppendVideoSegment(video);
michael@0: 
michael@0:   // The stream has stopped and reached the end of track.
michael@0:   if (aTrackEvents == MediaStreamListener::TRACK_EVENT_ENDED) {
michael@0:     LOG("[VideoTrackEncoder]: Receive TRACK_EVENT_ENDED .");
michael@0:     NotifyEndOfStream();
michael@0:   }
michael@0: 
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: VideoTrackEncoder::AppendVideoSegment(const VideoSegment& aSegment)
michael@0: {
michael@0:   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
michael@0: 
michael@0:   // Append all video segments from MediaStreamGraph, including null an
michael@0:   // non-null frames.
michael@0:   VideoSegment::ChunkIterator iter(const_cast<VideoSegment&>(aSegment));
michael@0:   while (!iter.IsEnded()) {
michael@0:     VideoChunk chunk = *iter;
michael@0:     nsRefPtr<layers::Image> image = chunk.mFrame.GetImage();
michael@0:     mRawSegment.AppendFrame(image.forget(), chunk.GetDuration(),
michael@0:                             chunk.mFrame.GetIntrinsicSize().ToIntSize());
michael@0:     iter.Next();
michael@0:   }
michael@0: 
michael@0:   if (mRawSegment.GetDuration() > 0) {
michael@0:     mReentrantMonitor.NotifyAll();
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: void
michael@0: VideoTrackEncoder::NotifyEndOfStream()
michael@0: {
michael@0:   // If source video track is muted till the end of encoding, initialize the
michael@0:   // encoder with default frame width, frame height, and track rate.
michael@0:   if (!mCanceled && !mInitialized) {
michael@0:     Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT,
michael@0:          DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT, DEFAULT_TRACK_RATE);
michael@0:   }
michael@0: 
michael@0:   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
michael@0:   mEndOfStream = true;
michael@0:   mReentrantMonitor.NotifyAll();
michael@0: }
michael@0: 
michael@0: void
michael@0: VideoTrackEncoder::CreateMutedFrame(nsTArray<uint8_t>* aOutputBuffer)
michael@0: {
michael@0:   NS_ENSURE_TRUE_VOID(aOutputBuffer);
michael@0: 
michael@0:   // Supports YUV420 image format only.
michael@0:   int yPlaneLen = mFrameWidth * mFrameHeight;
michael@0:   int cbcrPlaneLen = yPlaneLen / 2;
michael@0:   int frameLen = yPlaneLen + cbcrPlaneLen;
michael@0: 
michael@0:   aOutputBuffer->SetLength(frameLen);
michael@0:   // Fill Y plane.
michael@0:   memset(aOutputBuffer->Elements(), 0x10, yPlaneLen);
michael@0:   // Fill Cb/Cr planes.
michael@0:   memset(aOutputBuffer->Elements() + yPlaneLen, 0x80, cbcrPlaneLen);
michael@0: }
michael@0: 
michael@0: }