michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: #include "MediaEncoder.h"
michael@0: #include "MediaDecoder.h"
michael@0: #include "nsIPrincipal.h"
michael@0: #include "nsMimeTypes.h"
michael@0: #include "prlog.h"
michael@0: #include "mozilla/Preferences.h"
michael@0: 
michael@0: #include "OggWriter.h"
michael@0: #ifdef MOZ_OPUS
michael@0: #include "OpusTrackEncoder.h"
michael@0: 
michael@0: #endif
michael@0: 
michael@0: #ifdef MOZ_VORBIS
michael@0: #include "VorbisTrackEncoder.h"
michael@0: #endif
michael@0: #ifdef MOZ_WEBM_ENCODER
michael@0: #include "VorbisTrackEncoder.h"
michael@0: #include "VP8TrackEncoder.h"
michael@0: #include "WebMWriter.h"
michael@0: #endif
michael@0: #ifdef MOZ_OMX_ENCODER
michael@0: #include "OmxTrackEncoder.h"
michael@0: #include "ISOMediaWriter.h"
michael@0: #endif
michael@0: 
michael@0: #ifdef LOG
michael@0: #undef LOG
michael@0: #endif
michael@0: 
michael@0: #ifdef PR_LOGGING
michael@0: PRLogModuleInfo* gMediaEncoderLog;
michael@0: #define LOG(type, msg) PR_LOG(gMediaEncoderLog, type, msg)
michael@0: #else
michael@0: #define LOG(type, msg)
michael@0: #endif
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: void
michael@0: MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph,
michael@0:                                        TrackID aID,
michael@0:                                        TrackRate aTrackRate,
michael@0:                                        TrackTicks aTrackOffset,
michael@0:                                        uint32_t aTrackEvents,
michael@0:                                        const MediaSegment& aQueuedMedia)
michael@0: {
michael@0:   // Process the incoming raw track data from MediaStreamGraph, called on the
michael@0:   // thread of MediaStreamGraph.
michael@0:   if (mAudioEncoder && aQueuedMedia.GetType() == MediaSegment::AUDIO) {
michael@0:     mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID, aTrackRate,
michael@0:                                             aTrackOffset, aTrackEvents,
michael@0:                                             aQueuedMedia);
michael@0: 
michael@0:   } else if (mVideoEncoder && aQueuedMedia.GetType() == MediaSegment::VIDEO) {
michael@0:       mVideoEncoder->NotifyQueuedTrackChanges(aGraph, aID, aTrackRate,
michael@0:                                               aTrackOffset, aTrackEvents,
michael@0:                                               aQueuedMedia);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: MediaEncoder::NotifyRemoved(MediaStreamGraph* aGraph)
michael@0: {
michael@0:   // In case that MediaEncoder does not receive a TRACK_EVENT_ENDED event.
michael@0:   LOG(PR_LOG_DEBUG, ("NotifyRemoved in [MediaEncoder]."));
michael@0:   if (mAudioEncoder) {
michael@0:     mAudioEncoder->NotifyRemoved(aGraph);
michael@0:   }
michael@0:   if (mVideoEncoder) {
michael@0:     mVideoEncoder->NotifyRemoved(aGraph);
michael@0:   }
michael@0: 
michael@0: }
michael@0: 
michael@0: /* static */
michael@0: already_AddRefed<MediaEncoder>
michael@0: MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint8_t aTrackTypes)
michael@0: {
michael@0: #ifdef PR_LOGGING
michael@0:   if (!gMediaEncoderLog) {
michael@0:     gMediaEncoderLog = PR_NewLogModule("MediaEncoder");
michael@0:   }
michael@0: #endif
michael@0:   nsAutoPtr<ContainerWriter> writer;
michael@0:   nsAutoPtr<AudioTrackEncoder> audioEncoder;
michael@0:   nsAutoPtr<VideoTrackEncoder> videoEncoder;
michael@0:   nsRefPtr<MediaEncoder> encoder;
michael@0:   nsString mimeType;
michael@0:   if (!aTrackTypes) {
michael@0:     LOG(PR_LOG_ERROR, ("NO TrackTypes!!!"));
michael@0:     return nullptr;
michael@0:   }
michael@0: #ifdef MOZ_WEBM_ENCODER
michael@0:   else if (MediaEncoder::IsWebMEncoderEnabled() &&
michael@0:           (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
michael@0:           (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
michael@0:     if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK) {
michael@0:       audioEncoder = new VorbisTrackEncoder();
michael@0:       NS_ENSURE_TRUE(audioEncoder, nullptr);
michael@0:     }
michael@0:     videoEncoder = new VP8TrackEncoder();
michael@0:     writer = new WebMWriter(aTrackTypes);
michael@0:     NS_ENSURE_TRUE(writer, nullptr);
michael@0:     NS_ENSURE_TRUE(videoEncoder, nullptr);
michael@0:     mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
michael@0:   }
michael@0: #endif //MOZ_WEBM_ENCODER
michael@0: #ifdef MOZ_OMX_ENCODER
michael@0:   else if (MediaEncoder::IsOMXEncoderEnabled() &&
michael@0:           (aMIMEType.EqualsLiteral(VIDEO_MP4) ||
michael@0:           (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
michael@0:     if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK) {
michael@0:       audioEncoder = new OmxAACAudioTrackEncoder();
michael@0:       NS_ENSURE_TRUE(audioEncoder, nullptr);
michael@0:     }
michael@0:     videoEncoder = new OmxVideoTrackEncoder();
michael@0:     writer = new ISOMediaWriter(aTrackTypes);
michael@0:     NS_ENSURE_TRUE(writer, nullptr);
michael@0:     NS_ENSURE_TRUE(videoEncoder, nullptr);
michael@0:     mimeType = NS_LITERAL_STRING(VIDEO_MP4);
michael@0:   } else if (MediaEncoder::IsOMXEncoderEnabled() &&
michael@0:             (aMIMEType.EqualsLiteral(AUDIO_3GPP))) {
michael@0:     audioEncoder = new OmxAMRAudioTrackEncoder();
michael@0:     NS_ENSURE_TRUE(audioEncoder, nullptr);
michael@0: 
michael@0:     writer = new ISOMediaWriter(aTrackTypes, ISOMediaWriter::TYPE_FRAG_3GP);
michael@0:     NS_ENSURE_TRUE(writer, nullptr);
michael@0:     mimeType = NS_LITERAL_STRING(AUDIO_3GPP);
michael@0:   }
michael@0: #endif // MOZ_OMX_ENCODER
michael@0:   else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
michael@0:            (aMIMEType.EqualsLiteral(AUDIO_OGG) ||
michael@0:            (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK))) {
michael@0:     writer = new OggWriter();
michael@0:     audioEncoder = new OpusTrackEncoder();
michael@0:     NS_ENSURE_TRUE(writer, nullptr);
michael@0:     NS_ENSURE_TRUE(audioEncoder, nullptr);
michael@0:     mimeType = NS_LITERAL_STRING(AUDIO_OGG);
michael@0:   }
michael@0:   else {
michael@0:     LOG(PR_LOG_ERROR, ("Can not find any encoder to record this media stream"));
michael@0:     return nullptr;
michael@0:   }
michael@0:   LOG(PR_LOG_DEBUG, ("Create encoder result:a[%d] v[%d] w[%d] mimeType = %s.",
michael@0:                       audioEncoder != nullptr, videoEncoder != nullptr,
michael@0:                       writer != nullptr, mimeType.get()));
michael@0:   encoder = new MediaEncoder(writer.forget(), audioEncoder.forget(),
michael@0:                              videoEncoder.forget(), mimeType);
michael@0:   return encoder.forget();
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * GetEncodedData() runs as a state machine, starting with mState set to
michael@0:  * GET_METADDATA, the procedure should be as follow:
michael@0:  *
michael@0:  * While non-stop
michael@0:  *   If mState is GET_METADDATA
michael@0:  *     Get the meta data from audio/video encoder
michael@0:  *     If a meta data is generated
michael@0:  *       Get meta data from audio/video encoder
michael@0:  *       Set mState to ENCODE_TRACK
michael@0:  *       Return the final container data
michael@0:  *
michael@0:  *   If mState is ENCODE_TRACK
michael@0:  *     Get encoded track data from audio/video encoder
michael@0:  *     If a packet of track data is generated
michael@0:  *       Insert encoded track data into the container stream of writer
michael@0:  *       If the final container data is copied to aOutput
michael@0:  *         Return the copy of final container data
michael@0:  *       If this is the last packet of input stream
michael@0:  *         Set mState to ENCODE_DONE
michael@0:  *
michael@0:  *   If mState is ENCODE_DONE or ENCODE_ERROR
michael@0:  *     Stop the loop
michael@0:  */
michael@0: void
michael@0: MediaEncoder::GetEncodedData(nsTArray<nsTArray<uint8_t> >* aOutputBufs,
michael@0:                              nsAString& aMIMEType)
michael@0: {
michael@0:   MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:   aMIMEType = mMIMEType;
michael@0: 
michael@0:   bool reloop = true;
michael@0:   while (reloop) {
michael@0:     switch (mState) {
michael@0:     case ENCODE_METADDATA: {
michael@0:       LOG(PR_LOG_DEBUG, ("ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
michael@0:       nsresult rv = CopyMetadataToMuxer(mAudioEncoder.get());
michael@0:       if (NS_FAILED(rv)) {
michael@0:         LOG(PR_LOG_ERROR, ("Error! Fail to Set Audio Metadata"));
michael@0:         break;
michael@0:       }
michael@0:       rv = CopyMetadataToMuxer(mVideoEncoder.get());
michael@0:       if (NS_FAILED(rv)) {
michael@0:         LOG(PR_LOG_ERROR, ("Error! Fail to Set Video Metadata"));
michael@0:         break;
michael@0:       }
michael@0: 
michael@0:       rv = mWriter->GetContainerData(aOutputBufs,
michael@0:                                      ContainerWriter::GET_HEADER);
michael@0:       if (NS_FAILED(rv)) {
michael@0:        LOG(PR_LOG_ERROR,("Error! writer fail to generate header!"));
michael@0:        mState = ENCODE_ERROR;
michael@0:        break;
michael@0:       }
michael@0:       LOG(PR_LOG_DEBUG, ("Finish ENCODE_METADDATA TimeStamp = %f", GetEncodeTimeStamp()));
michael@0:       mState = ENCODE_TRACK;
michael@0:       break;
michael@0:     }
michael@0: 
michael@0:     case ENCODE_TRACK: {
michael@0:       LOG(PR_LOG_DEBUG, ("ENCODE_TRACK TimeStamp = %f", GetEncodeTimeStamp()));
michael@0:       EncodedFrameContainer encodedData;
michael@0:       nsresult rv = NS_OK;
michael@0:       rv = WriteEncodedDataToMuxer(mAudioEncoder.get());
michael@0:       if (NS_FAILED(rv)) {
michael@0:         LOG(PR_LOG_ERROR, ("Error! Fail to write audio encoder data to muxer"));
michael@0:         break;
michael@0:       }
michael@0:       LOG(PR_LOG_DEBUG, ("Audio encoded TimeStamp = %f", GetEncodeTimeStamp()));
michael@0:       rv = WriteEncodedDataToMuxer(mVideoEncoder.get());
michael@0:       if (NS_FAILED(rv)) {
michael@0:         LOG(PR_LOG_ERROR, ("Fail to write video encoder data to muxer"));
michael@0:         break;
michael@0:       }
michael@0:       LOG(PR_LOG_DEBUG, ("Video encoded TimeStamp = %f", GetEncodeTimeStamp()));
michael@0:       // In audio only or video only case, let unavailable track's flag to be true.
michael@0:       bool isAudioCompleted = (mAudioEncoder && mAudioEncoder->IsEncodingComplete()) || !mAudioEncoder;
michael@0:       bool isVideoCompleted = (mVideoEncoder && mVideoEncoder->IsEncodingComplete()) || !mVideoEncoder;
michael@0:       rv = mWriter->GetContainerData(aOutputBufs,
michael@0:                                      isAudioCompleted && isVideoCompleted ?
michael@0:                                      ContainerWriter::FLUSH_NEEDED : 0);
michael@0:       if (NS_SUCCEEDED(rv)) {
michael@0:         // Successfully get the copy of final container data from writer.
michael@0:         reloop = false;
michael@0:       }
michael@0:       mState = (mWriter->IsWritingComplete()) ? ENCODE_DONE : ENCODE_TRACK;
michael@0:       LOG(PR_LOG_DEBUG, ("END ENCODE_TRACK TimeStamp = %f "
michael@0:           "mState = %d aComplete %d vComplete %d",
michael@0:           GetEncodeTimeStamp(), mState, isAudioCompleted, isVideoCompleted));
michael@0:       break;
michael@0:     }
michael@0: 
michael@0:     case ENCODE_DONE:
michael@0:     case ENCODE_ERROR:
michael@0:       LOG(PR_LOG_DEBUG, ("MediaEncoder has been shutdown."));
michael@0:       mShutdown = true;
michael@0:       reloop = false;
michael@0:       break;
michael@0:     default:
michael@0:       MOZ_CRASH("Invalid encode state");
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaEncoder::WriteEncodedDataToMuxer(TrackEncoder *aTrackEncoder)
michael@0: {
michael@0:   if (aTrackEncoder == nullptr) {
michael@0:     return NS_OK;
michael@0:   }
michael@0:   if (aTrackEncoder->IsEncodingComplete()) {
michael@0:     return NS_OK;
michael@0:   }
michael@0:   EncodedFrameContainer encodedVideoData;
michael@0:   nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData);
michael@0:   if (NS_FAILED(rv)) {
michael@0:     // Encoding might be canceled.
michael@0:     LOG(PR_LOG_ERROR, ("Error! Fail to get encoded data from video encoder."));
michael@0:     mState = ENCODE_ERROR;
michael@0:     return rv;
michael@0:   }
michael@0:   rv = mWriter->WriteEncodedTrack(encodedVideoData,
michael@0:                                   aTrackEncoder->IsEncodingComplete() ?
michael@0:                                   ContainerWriter::END_OF_STREAM : 0);
michael@0:   if (NS_FAILED(rv)) {
michael@0:     LOG(PR_LOG_ERROR, ("Error! Fail to write encoded video track to the media container."));
michael@0:     mState = ENCODE_ERROR;
michael@0:   }
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: MediaEncoder::CopyMetadataToMuxer(TrackEncoder *aTrackEncoder)
michael@0: {
michael@0:   if (aTrackEncoder == nullptr) {
michael@0:     return NS_OK;
michael@0:   }
michael@0:   nsRefPtr<TrackMetadataBase> meta = aTrackEncoder->GetMetadata();
michael@0:   if (meta == nullptr) {
michael@0:     LOG(PR_LOG_ERROR, ("Error! metadata = null"));
michael@0:     mState = ENCODE_ERROR;
michael@0:     return NS_ERROR_ABORT;
michael@0:   }
michael@0: 
michael@0:   nsresult rv = mWriter->SetMetadata(meta);
michael@0:   if (NS_FAILED(rv)) {
michael@0:    LOG(PR_LOG_ERROR, ("Error! SetMetadata fail"));
michael@0:    mState = ENCODE_ERROR;
michael@0:   }
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: #ifdef MOZ_WEBM_ENCODER
michael@0: bool
michael@0: MediaEncoder::IsWebMEncoderEnabled()
michael@0: {
michael@0:   return Preferences::GetBool("media.encoder.webm.enabled");
michael@0: }
michael@0: #endif
michael@0: 
michael@0: #ifdef MOZ_OMX_ENCODER
michael@0: bool
michael@0: MediaEncoder::IsOMXEncoderEnabled()
michael@0: {
michael@0:   return Preferences::GetBool("media.encoder.omx.enabled");
michael@0: }
michael@0: #endif
michael@0: 
michael@0: }