michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef MEDIAENGINEWEBRTC_H_
michael@0: #define MEDIAENGINEWEBRTC_H_
michael@0: 
michael@0: #include "prcvar.h"
michael@0: #include "prthread.h"
michael@0: #include "nsIThread.h"
michael@0: #include "nsIRunnable.h"
michael@0: 
michael@0: #include "mozilla/Mutex.h"
michael@0: #include "mozilla/Monitor.h"
michael@0: #include "nsCOMPtr.h"
michael@0: #include "nsDOMFile.h"
michael@0: #include "nsThreadUtils.h"
michael@0: #include "DOMMediaStream.h"
michael@0: #include "nsDirectoryServiceDefs.h"
michael@0: #include "nsComponentManagerUtils.h"
michael@0: #include "nsRefPtrHashtable.h"
michael@0: 
michael@0: #include "VideoUtils.h"
michael@0: #include "MediaEngine.h"
michael@0: #include "VideoSegment.h"
michael@0: #include "AudioSegment.h"
michael@0: #include "StreamBuffer.h"
michael@0: #include "MediaStreamGraph.h"
michael@0: 
michael@0: #include "MediaEngineWrapper.h"
michael@0: 
michael@0: // WebRTC library includes follow
michael@0: 
michael@0: // Audio Engine
michael@0: #include "webrtc/voice_engine/include/voe_base.h"
michael@0: #include "webrtc/voice_engine/include/voe_codec.h"
michael@0: #include "webrtc/voice_engine/include/voe_hardware.h"
michael@0: #include "webrtc/voice_engine/include/voe_network.h"
michael@0: #include "webrtc/voice_engine/include/voe_audio_processing.h"
michael@0: #include "webrtc/voice_engine/include/voe_volume_control.h"
michael@0: #include "webrtc/voice_engine/include/voe_external_media.h"
michael@0: #include "webrtc/voice_engine/include/voe_audio_processing.h"
michael@0: #include "webrtc/voice_engine/include/voe_call_report.h"
michael@0: 
michael@0: // Video Engine
michael@0: #include "webrtc/video_engine/include/vie_base.h"
michael@0: #include "webrtc/video_engine/include/vie_codec.h"
michael@0: #include "webrtc/video_engine/include/vie_render.h"
michael@0: #include "webrtc/video_engine/include/vie_capture.h"
michael@0: #ifdef MOZ_B2G_CAMERA
michael@0: #include "CameraControlListener.h"
michael@0: #include "ICameraControl.h"
michael@0: #include "ImageContainer.h"
michael@0: #include "nsGlobalWindow.h"
michael@0: #include "prprf.h"
michael@0: #include "mozilla/Hal.h"
michael@0: #endif
michael@0: 
michael@0: #include "NullTransport.h"
michael@0: #include "AudioOutputObserver.h"
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: #ifdef MOZ_B2G_CAMERA
michael@0: class CameraAllocateRunnable;
michael@0: class GetCameraNameRunnable;
michael@0: #endif
michael@0: 
michael@0: /**
michael@0:  * The WebRTC implementation of the MediaEngine interface.
michael@0:  *
michael@0:  * On B2G platform, member data may accessed from different thread after construction:
michael@0:  *
michael@0:  * MediaThread:
michael@0:  *   mState, mImage, mWidth, mHeight, mCapability, mPrefs, mDeviceName, mUniqueId, mInitDone,
michael@0:  *   mImageContainer, mSources, mState, mImage
michael@0:  *
michael@0:  * MainThread:
michael@0:  *   mCaptureIndex, mLastCapture, mState,  mWidth, mHeight,
michael@0:  *
michael@0:  * Where mWidth, mHeight, mImage are protected by mMonitor
michael@0:  *       mState is protected by mCallbackMonitor
michael@0:  * Other variable is accessed only from single thread
michael@0:  */
michael@0: class MediaEngineWebRTCVideoSource : public MediaEngineVideoSource
michael@0:                                    , public nsRunnable
michael@0: #ifdef MOZ_B2G_CAMERA
michael@0:                                    , public CameraControlListener
michael@0:                                    , public mozilla::hal::ScreenConfigurationObserver
michael@0: #else
michael@0:                                    , public webrtc::ExternalRenderer
michael@0: #endif
michael@0: {
michael@0: public:
michael@0: #ifdef MOZ_B2G_CAMERA
michael@0:   MediaEngineWebRTCVideoSource(int aIndex)
michael@0:     : mCameraControl(nullptr)
michael@0:     , mCallbackMonitor("WebRTCCamera.CallbackMonitor")
michael@0:     , mRotation(0)
michael@0:     , mBackCamera(false)
michael@0:     , mCaptureIndex(aIndex)
michael@0:     , mMonitor("WebRTCCamera.Monitor")
michael@0:     , mWidth(0)
michael@0:     , mHeight(0)
michael@0:     , mInitDone(false)
michael@0:     , mInSnapshotMode(false)
michael@0:     , mSnapshotPath(nullptr)
michael@0:   {
michael@0:     mState = kReleased;
michael@0:     Init();
michael@0:   }
michael@0: #else
michael@0:   // ViEExternalRenderer.
michael@0:   virtual int FrameSizeChange(unsigned int, unsigned int, unsigned int);
michael@0:   virtual int DeliverFrame(unsigned char*,int, uint32_t , int64_t,
michael@0:                            void *handle);
michael@0:   /**
michael@0:    * Does DeliverFrame() support a null buffer and non-null handle
michael@0:    * (video texture)?
michael@0:    * XXX Investigate!  Especially for Android/B2G
michael@0:    */
michael@0:   virtual bool IsTextureSupported() { return false; }
michael@0: 
michael@0:   MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr, int aIndex)
michael@0:     : mVideoEngine(aVideoEnginePtr)
michael@0:     , mCaptureIndex(aIndex)
michael@0:     , mFps(-1)
michael@0:     , mMinFps(-1)
michael@0:     , mMonitor("WebRTCCamera.Monitor")
michael@0:     , mWidth(0)
michael@0:     , mHeight(0)
michael@0:     , mInitDone(false)
michael@0:     , mInSnapshotMode(false)
michael@0:     , mSnapshotPath(nullptr) {
michael@0:     MOZ_ASSERT(aVideoEnginePtr);
michael@0:     mState = kReleased;
michael@0:     Init();
michael@0:   }
michael@0: #endif
michael@0: 
michael@0:   ~MediaEngineWebRTCVideoSource() { Shutdown(); }
michael@0: 
michael@0:   virtual void GetName(nsAString&);
michael@0:   virtual void GetUUID(nsAString&);
michael@0:   virtual nsresult Allocate(const VideoTrackConstraintsN &aConstraints,
michael@0:                             const MediaEnginePrefs &aPrefs);
michael@0:   virtual nsresult Deallocate();
michael@0:   virtual nsresult Start(SourceMediaStream*, TrackID);
michael@0:   virtual nsresult Stop(SourceMediaStream*, TrackID);
michael@0:   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
michael@0:   virtual nsresult Config(bool aEchoOn, uint32_t aEcho,
michael@0:                           bool aAgcOn, uint32_t aAGC,
michael@0:                           bool aNoiseOn, uint32_t aNoise,
michael@0:                           int32_t aPlayoutDelay) { return NS_OK; };
michael@0:   virtual void NotifyPull(MediaStreamGraph* aGraph,
michael@0:                           SourceMediaStream *aSource,
michael@0:                           TrackID aId,
michael@0:                           StreamTime aDesiredTime,
michael@0:                           TrackTicks &aLastEndTime);
michael@0: 
michael@0:   virtual bool IsFake() {
michael@0:     return false;
michael@0:   }
michael@0: 
michael@0: #ifndef MOZ_B2G_CAMERA
michael@0:   NS_DECL_THREADSAFE_ISUPPORTS
michael@0: #else
michael@0:   // We are subclassed from CameraControlListener, which implements a
michael@0:   // threadsafe reference-count for us.
michael@0:   NS_DECL_ISUPPORTS_INHERITED
michael@0: 
michael@0:   void OnHardwareStateChange(HardwareState aState);
michael@0:   bool OnNewPreviewFrame(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight);
michael@0:   void OnError(CameraErrorContext aContext, CameraError aError);
michael@0:   void OnTakePictureComplete(uint8_t* aData, uint32_t aLength, const nsAString& aMimeType);
michael@0: 
michael@0:   void AllocImpl();
michael@0:   void DeallocImpl();
michael@0:   void StartImpl(webrtc::CaptureCapability aCapability);
michael@0:   void StopImpl();
michael@0:   void SnapshotImpl();
michael@0:   void RotateImage(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight);
michael@0:   void Notify(const mozilla::hal::ScreenConfiguration& aConfiguration);
michael@0: #endif
michael@0: 
michael@0:   // This runnable is for creating a temporary file on the main thread.
michael@0:   NS_IMETHODIMP
michael@0:   Run()
michael@0:   {
michael@0:     nsCOMPtr<nsIFile> tmp;
michael@0:     nsresult rv = NS_GetSpecialDirectory(NS_OS_TEMP_DIR, getter_AddRefs(tmp));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     tmp->Append(NS_LITERAL_STRING("webrtc_snapshot.jpeg"));
michael@0:     rv = tmp->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 0600);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     mSnapshotPath = new nsString();
michael@0:     rv = tmp->GetPath(*mSnapshotPath);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     return NS_OK;
michael@0:   }
michael@0: 
michael@0: private:
michael@0:   static const unsigned int KMaxDeviceNameLength = 128;
michael@0:   static const unsigned int KMaxUniqueIdLength = 256;
michael@0: 
michael@0:   // Initialize the needed Video engine interfaces.
michael@0:   void Init();
michael@0:   void Shutdown();
michael@0: 
michael@0:   // Engine variables.
michael@0: #ifdef MOZ_B2G_CAMERA
michael@0:   nsRefPtr<ICameraControl> mCameraControl;
michael@0:   mozilla::ReentrantMonitor mCallbackMonitor; // Monitor for camera callback handling
michael@0:   nsRefPtr<nsIDOMFile> mLastCapture;
michael@0:   int mRotation;
michael@0:   int mCameraAngle; // See dom/base/ScreenOrientation.h
michael@0:   bool mBackCamera;
michael@0: #else
michael@0:   webrtc::VideoEngine* mVideoEngine; // Weak reference, don't free.
michael@0:   webrtc::ViEBase* mViEBase;
michael@0:   webrtc::ViECapture* mViECapture;
michael@0:   webrtc::ViERender* mViERender;
michael@0: #endif
michael@0:   webrtc::CaptureCapability mCapability; // Doesn't work on OS X.
michael@0: 
michael@0:   int mCaptureIndex;
michael@0:   int mFps; // Track rate (30 fps by default)
michael@0:   int mMinFps; // Min rate we want to accept
michael@0: 
michael@0:   // mMonitor protects mImage access/changes, and transitions of mState
michael@0:   // from kStarted to kStopped (which are combined with EndTrack() and
michael@0:   // image changes).  Note that mSources is not accessed from other threads
michael@0:   // for video and is not protected.
michael@0:   Monitor mMonitor; // Monitor for processing WebRTC frames.
michael@0:   int mWidth, mHeight;
michael@0:   nsRefPtr<layers::Image> mImage;
michael@0:   nsRefPtr<layers::ImageContainer> mImageContainer;
michael@0: 
michael@0:   nsTArray<SourceMediaStream *> mSources; // When this goes empty, we shut down HW
michael@0: 
michael@0:   bool mInitDone;
michael@0:   bool mInSnapshotMode;
michael@0:   nsString* mSnapshotPath;
michael@0: 
michael@0:   nsString mDeviceName;
michael@0:   nsString mUniqueId;
michael@0: 
michael@0:   void ChooseCapability(const VideoTrackConstraintsN &aConstraints,
michael@0:                         const MediaEnginePrefs &aPrefs);
michael@0: 
michael@0:   void GuessCapability(const VideoTrackConstraintsN &aConstraints,
michael@0:                        const MediaEnginePrefs &aPrefs);
michael@0: };
michael@0: 
michael@0: class MediaEngineWebRTCAudioSource : public MediaEngineAudioSource,
michael@0:                                      public webrtc::VoEMediaProcess
michael@0: {
michael@0: public:
michael@0:   MediaEngineWebRTCAudioSource(webrtc::VoiceEngine* aVoiceEnginePtr, int aIndex,
michael@0:     const char* name, const char* uuid)
michael@0:     : mVoiceEngine(aVoiceEnginePtr)
michael@0:     , mMonitor("WebRTCMic.Monitor")
michael@0:     , mCapIndex(aIndex)
michael@0:     , mChannel(-1)
michael@0:     , mInitDone(false)
michael@0:     , mStarted(false)
michael@0:     , mSamples(0)
michael@0:     , mEchoOn(false), mAgcOn(false), mNoiseOn(false)
michael@0:     , mEchoCancel(webrtc::kEcDefault)
michael@0:     , mAGC(webrtc::kAgcDefault)
michael@0:     , mNoiseSuppress(webrtc::kNsDefault)
michael@0:     , mPlayoutDelay(0)
michael@0:     , mNullTransport(nullptr) {
michael@0:     MOZ_ASSERT(aVoiceEnginePtr);
michael@0:     mState = kReleased;
michael@0:     mDeviceName.Assign(NS_ConvertUTF8toUTF16(name));
michael@0:     mDeviceUUID.Assign(NS_ConvertUTF8toUTF16(uuid));
michael@0:     Init();
michael@0:   }
michael@0:   ~MediaEngineWebRTCAudioSource() { Shutdown(); }
michael@0: 
michael@0:   virtual void GetName(nsAString&);
michael@0:   virtual void GetUUID(nsAString&);
michael@0: 
michael@0:   virtual nsresult Allocate(const AudioTrackConstraintsN &aConstraints,
michael@0:                             const MediaEnginePrefs &aPrefs);
michael@0:   virtual nsresult Deallocate();
michael@0:   virtual nsresult Start(SourceMediaStream*, TrackID);
michael@0:   virtual nsresult Stop(SourceMediaStream*, TrackID);
michael@0:   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
michael@0:   virtual nsresult Config(bool aEchoOn, uint32_t aEcho,
michael@0:                           bool aAgcOn, uint32_t aAGC,
michael@0:                           bool aNoiseOn, uint32_t aNoise,
michael@0:                           int32_t aPlayoutDelay);
michael@0: 
michael@0:   virtual void NotifyPull(MediaStreamGraph* aGraph,
michael@0:                           SourceMediaStream *aSource,
michael@0:                           TrackID aId,
michael@0:                           StreamTime aDesiredTime,
michael@0:                           TrackTicks &aLastEndTime);
michael@0: 
michael@0:   virtual bool IsFake() {
michael@0:     return false;
michael@0:   }
michael@0: 
michael@0:   // VoEMediaProcess.
michael@0:   void Process(int channel, webrtc::ProcessingTypes type,
michael@0:                int16_t audio10ms[], int length,
michael@0:                int samplingFreq, bool isStereo);
michael@0: 
michael@0:   NS_DECL_THREADSAFE_ISUPPORTS
michael@0: 
michael@0: private:
michael@0:   static const unsigned int KMaxDeviceNameLength = 128;
michael@0:   static const unsigned int KMaxUniqueIdLength = 256;
michael@0: 
michael@0:   void Init();
michael@0:   void Shutdown();
michael@0: 
michael@0:   webrtc::VoiceEngine* mVoiceEngine;
michael@0:   ScopedCustomReleasePtr<webrtc::VoEBase> mVoEBase;
michael@0:   ScopedCustomReleasePtr<webrtc::VoEExternalMedia> mVoERender;
michael@0:   ScopedCustomReleasePtr<webrtc::VoENetwork> mVoENetwork;
michael@0:   ScopedCustomReleasePtr<webrtc::VoEAudioProcessing> mVoEProcessing;
michael@0:   ScopedCustomReleasePtr<webrtc::VoECallReport> mVoECallReport;
michael@0: 
michael@0:   // mMonitor protects mSources[] access/changes, and transitions of mState
michael@0:   // from kStarted to kStopped (which are combined with EndTrack()).
michael@0:   // mSources[] is accessed from webrtc threads.
michael@0:   Monitor mMonitor;
michael@0:   nsTArray<SourceMediaStream *> mSources; // When this goes empty, we shut down HW
michael@0: 
michael@0:   int mCapIndex;
michael@0:   int mChannel;
michael@0:   TrackID mTrackID;
michael@0:   bool mInitDone;
michael@0:   bool mStarted;
michael@0:   int mSamples; // int to avoid conversions when comparing/etc to samplingFreq & length
michael@0: 
michael@0:   nsString mDeviceName;
michael@0:   nsString mDeviceUUID;
michael@0: 
michael@0:   bool mEchoOn, mAgcOn, mNoiseOn;
michael@0:   webrtc::EcModes  mEchoCancel;
michael@0:   webrtc::AgcModes mAGC;
michael@0:   webrtc::NsModes  mNoiseSuppress;
michael@0:   int32_t mPlayoutDelay;
michael@0: 
michael@0:   NullTransport *mNullTransport;
michael@0: };
michael@0: 
michael@0: class MediaEngineWebRTC : public MediaEngine
michael@0: {
michael@0: public:
michael@0:   MediaEngineWebRTC(MediaEnginePrefs &aPrefs);
michael@0: 
michael@0:   // Clients should ensure to clean-up sources video/audio sources
michael@0:   // before invoking Shutdown on this class.
michael@0:   void Shutdown();
michael@0: 
michael@0:   virtual void EnumerateVideoDevices(nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
michael@0:   virtual void EnumerateAudioDevices(nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
michael@0: 
michael@0: private:
michael@0:   ~MediaEngineWebRTC() {
michael@0:     Shutdown();
michael@0: #ifdef MOZ_B2G_CAMERA
michael@0:     AsyncLatencyLogger::Get()->Release();
michael@0: #endif
michael@0:     // XXX
michael@0:     gFarendObserver = nullptr;
michael@0:   }
michael@0: 
michael@0:   Mutex mMutex;
michael@0:   // protected with mMutex:
michael@0: 
michael@0:   webrtc::VideoEngine* mVideoEngine;
michael@0:   webrtc::VoiceEngine* mVoiceEngine;
michael@0: 
michael@0:   // Need this to avoid unneccesary WebRTC calls while enumerating.
michael@0:   bool mVideoEngineInit;
michael@0:   bool mAudioEngineInit;
michael@0:   bool mHasTabVideoSource;
michael@0: 
michael@0:   // Store devices we've already seen in a hashtable for quick return.
michael@0:   // Maps UUID to MediaEngineSource (one set for audio, one for video).
michael@0:   nsRefPtrHashtable<nsStringHashKey, MediaEngineWebRTCVideoSource > mVideoSources;
michael@0:   nsRefPtrHashtable<nsStringHashKey, MediaEngineWebRTCAudioSource > mAudioSources;
michael@0: };
michael@0: 
michael@0: }
michael@0: 
michael@0: #endif /* NSMEDIAENGINEWEBRTC_H_ */