1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/content/media/webrtc/MediaEngineWebRTC.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,396 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.6 + * You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +#ifndef MEDIAENGINEWEBRTC_H_
1.9 +#define MEDIAENGINEWEBRTC_H_
1.10 +
1.11 +#include "prcvar.h"
1.12 +#include "prthread.h"
1.13 +#include "nsIThread.h"
1.14 +#include "nsIRunnable.h"
1.15 +
1.16 +#include "mozilla/Mutex.h"
1.17 +#include "mozilla/Monitor.h"
1.18 +#include "nsCOMPtr.h"
1.19 +#include "nsDOMFile.h"
1.20 +#include "nsThreadUtils.h"
1.21 +#include "DOMMediaStream.h"
1.22 +#include "nsDirectoryServiceDefs.h"
1.23 +#include "nsComponentManagerUtils.h"
1.24 +#include "nsRefPtrHashtable.h"
1.25 +
1.26 +#include "VideoUtils.h"
1.27 +#include "MediaEngine.h"
1.28 +#include "VideoSegment.h"
1.29 +#include "AudioSegment.h"
1.30 +#include "StreamBuffer.h"
1.31 +#include "MediaStreamGraph.h"
1.32 +
1.33 +#include "MediaEngineWrapper.h"
1.34 +
1.35 +// WebRTC library includes follow
1.36 +
1.37 +// Audio Engine
1.38 +#include "webrtc/voice_engine/include/voe_base.h"
1.39 +#include "webrtc/voice_engine/include/voe_codec.h"
1.40 +#include "webrtc/voice_engine/include/voe_hardware.h"
1.41 +#include "webrtc/voice_engine/include/voe_network.h"
1.42 +#include "webrtc/voice_engine/include/voe_audio_processing.h"
1.43 +#include "webrtc/voice_engine/include/voe_volume_control.h"
1.44 +#include "webrtc/voice_engine/include/voe_external_media.h"
1.45 +#include "webrtc/voice_engine/include/voe_audio_processing.h"
1.46 +#include "webrtc/voice_engine/include/voe_call_report.h"
1.47 +
1.48 +// Video Engine
1.49 +#include "webrtc/video_engine/include/vie_base.h"
1.50 +#include "webrtc/video_engine/include/vie_codec.h"
1.51 +#include "webrtc/video_engine/include/vie_render.h"
1.52 +#include "webrtc/video_engine/include/vie_capture.h"
1.53 +#ifdef MOZ_B2G_CAMERA
1.54 +#include "CameraControlListener.h"
1.55 +#include "ICameraControl.h"
1.56 +#include "ImageContainer.h"
1.57 +#include "nsGlobalWindow.h"
1.58 +#include "prprf.h"
1.59 +#include "mozilla/Hal.h"
1.60 +#endif
1.61 +
1.62 +#include "NullTransport.h"
1.63 +#include "AudioOutputObserver.h"
1.64 +
1.65 +namespace mozilla {
1.66 +
1.67 +#ifdef MOZ_B2G_CAMERA
1.68 +class CameraAllocateRunnable;
1.69 +class GetCameraNameRunnable;
1.70 +#endif
1.71 +
1.72 +/**
1.73 + * The WebRTC implementation of the MediaEngine interface.
1.74 + *
1.75 + * On B2G platform, member data may accessed from different thread after construction:
1.76 + *
1.77 + * MediaThread:
1.78 + * mState, mImage, mWidth, mHeight, mCapability, mPrefs, mDeviceName, mUniqueId, mInitDone,
1.79 + * mImageContainer, mSources, mState, mImage
1.80 + *
1.81 + * MainThread:
1.82 + * mCaptureIndex, mLastCapture, mState, mWidth, mHeight,
1.83 + *
1.84 + * Where mWidth, mHeight, mImage are protected by mMonitor
1.85 + * mState is protected by mCallbackMonitor
1.86 + * Other variable is accessed only from single thread
1.87 + */
1.88 +class MediaEngineWebRTCVideoSource : public MediaEngineVideoSource
1.89 + , public nsRunnable
1.90 +#ifdef MOZ_B2G_CAMERA
1.91 + , public CameraControlListener
1.92 + , public mozilla::hal::ScreenConfigurationObserver
1.93 +#else
1.94 + , public webrtc::ExternalRenderer
1.95 +#endif
1.96 +{
1.97 +public:
1.98 +#ifdef MOZ_B2G_CAMERA
1.99 + MediaEngineWebRTCVideoSource(int aIndex)
1.100 + : mCameraControl(nullptr)
1.101 + , mCallbackMonitor("WebRTCCamera.CallbackMonitor")
1.102 + , mRotation(0)
1.103 + , mBackCamera(false)
1.104 + , mCaptureIndex(aIndex)
1.105 + , mMonitor("WebRTCCamera.Monitor")
1.106 + , mWidth(0)
1.107 + , mHeight(0)
1.108 + , mInitDone(false)
1.109 + , mInSnapshotMode(false)
1.110 + , mSnapshotPath(nullptr)
1.111 + {
1.112 + mState = kReleased;
1.113 + Init();
1.114 + }
1.115 +#else
1.116 + // ViEExternalRenderer.
1.117 + virtual int FrameSizeChange(unsigned int, unsigned int, unsigned int);
1.118 + virtual int DeliverFrame(unsigned char*,int, uint32_t , int64_t,
1.119 + void *handle);
1.120 + /**
1.121 + * Does DeliverFrame() support a null buffer and non-null handle
1.122 + * (video texture)?
1.123 + * XXX Investigate! Especially for Android/B2G
1.124 + */
1.125 + virtual bool IsTextureSupported() { return false; }
1.126 +
1.127 + MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr, int aIndex)
1.128 + : mVideoEngine(aVideoEnginePtr)
1.129 + , mCaptureIndex(aIndex)
1.130 + , mFps(-1)
1.131 + , mMinFps(-1)
1.132 + , mMonitor("WebRTCCamera.Monitor")
1.133 + , mWidth(0)
1.134 + , mHeight(0)
1.135 + , mInitDone(false)
1.136 + , mInSnapshotMode(false)
1.137 + , mSnapshotPath(nullptr) {
1.138 + MOZ_ASSERT(aVideoEnginePtr);
1.139 + mState = kReleased;
1.140 + Init();
1.141 + }
1.142 +#endif
1.143 +
1.144 + ~MediaEngineWebRTCVideoSource() { Shutdown(); }
1.145 +
1.146 + virtual void GetName(nsAString&);
1.147 + virtual void GetUUID(nsAString&);
1.148 + virtual nsresult Allocate(const VideoTrackConstraintsN &aConstraints,
1.149 + const MediaEnginePrefs &aPrefs);
1.150 + virtual nsresult Deallocate();
1.151 + virtual nsresult Start(SourceMediaStream*, TrackID);
1.152 + virtual nsresult Stop(SourceMediaStream*, TrackID);
1.153 + virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
1.154 + virtual nsresult Config(bool aEchoOn, uint32_t aEcho,
1.155 + bool aAgcOn, uint32_t aAGC,
1.156 + bool aNoiseOn, uint32_t aNoise,
1.157 + int32_t aPlayoutDelay) { return NS_OK; };
1.158 + virtual void NotifyPull(MediaStreamGraph* aGraph,
1.159 + SourceMediaStream *aSource,
1.160 + TrackID aId,
1.161 + StreamTime aDesiredTime,
1.162 + TrackTicks &aLastEndTime);
1.163 +
1.164 + virtual bool IsFake() {
1.165 + return false;
1.166 + }
1.167 +
1.168 +#ifndef MOZ_B2G_CAMERA
1.169 + NS_DECL_THREADSAFE_ISUPPORTS
1.170 +#else
1.171 + // We are subclassed from CameraControlListener, which implements a
1.172 + // threadsafe reference-count for us.
1.173 + NS_DECL_ISUPPORTS_INHERITED
1.174 +
1.175 + void OnHardwareStateChange(HardwareState aState);
1.176 + bool OnNewPreviewFrame(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight);
1.177 + void OnError(CameraErrorContext aContext, CameraError aError);
1.178 + void OnTakePictureComplete(uint8_t* aData, uint32_t aLength, const nsAString& aMimeType);
1.179 +
1.180 + void AllocImpl();
1.181 + void DeallocImpl();
1.182 + void StartImpl(webrtc::CaptureCapability aCapability);
1.183 + void StopImpl();
1.184 + void SnapshotImpl();
1.185 + void RotateImage(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight);
1.186 + void Notify(const mozilla::hal::ScreenConfiguration& aConfiguration);
1.187 +#endif
1.188 +
1.189 + // This runnable is for creating a temporary file on the main thread.
1.190 + NS_IMETHODIMP
1.191 + Run()
1.192 + {
1.193 + nsCOMPtr<nsIFile> tmp;
1.194 + nsresult rv = NS_GetSpecialDirectory(NS_OS_TEMP_DIR, getter_AddRefs(tmp));
1.195 + NS_ENSURE_SUCCESS(rv, rv);
1.196 +
1.197 + tmp->Append(NS_LITERAL_STRING("webrtc_snapshot.jpeg"));
1.198 + rv = tmp->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 0600);
1.199 + NS_ENSURE_SUCCESS(rv, rv);
1.200 +
1.201 + mSnapshotPath = new nsString();
1.202 + rv = tmp->GetPath(*mSnapshotPath);
1.203 + NS_ENSURE_SUCCESS(rv, rv);
1.204 +
1.205 + return NS_OK;
1.206 + }
1.207 +
1.208 +private:
1.209 + static const unsigned int KMaxDeviceNameLength = 128;
1.210 + static const unsigned int KMaxUniqueIdLength = 256;
1.211 +
1.212 + // Initialize the needed Video engine interfaces.
1.213 + void Init();
1.214 + void Shutdown();
1.215 +
1.216 + // Engine variables.
1.217 +#ifdef MOZ_B2G_CAMERA
1.218 + nsRefPtr<ICameraControl> mCameraControl;
1.219 + mozilla::ReentrantMonitor mCallbackMonitor; // Monitor for camera callback handling
1.220 + nsRefPtr<nsIDOMFile> mLastCapture;
1.221 + int mRotation;
1.222 + int mCameraAngle; // See dom/base/ScreenOrientation.h
1.223 + bool mBackCamera;
1.224 +#else
1.225 + webrtc::VideoEngine* mVideoEngine; // Weak reference, don't free.
1.226 + webrtc::ViEBase* mViEBase;
1.227 + webrtc::ViECapture* mViECapture;
1.228 + webrtc::ViERender* mViERender;
1.229 +#endif
1.230 + webrtc::CaptureCapability mCapability; // Doesn't work on OS X.
1.231 +
1.232 + int mCaptureIndex;
1.233 + int mFps; // Track rate (30 fps by default)
1.234 + int mMinFps; // Min rate we want to accept
1.235 +
1.236 + // mMonitor protects mImage access/changes, and transitions of mState
1.237 + // from kStarted to kStopped (which are combined with EndTrack() and
1.238 + // image changes). Note that mSources is not accessed from other threads
1.239 + // for video and is not protected.
1.240 + Monitor mMonitor; // Monitor for processing WebRTC frames.
1.241 + int mWidth, mHeight;
1.242 + nsRefPtr<layers::Image> mImage;
1.243 + nsRefPtr<layers::ImageContainer> mImageContainer;
1.244 +
1.245 + nsTArray<SourceMediaStream *> mSources; // When this goes empty, we shut down HW
1.246 +
1.247 + bool mInitDone;
1.248 + bool mInSnapshotMode;
1.249 + nsString* mSnapshotPath;
1.250 +
1.251 + nsString mDeviceName;
1.252 + nsString mUniqueId;
1.253 +
1.254 + void ChooseCapability(const VideoTrackConstraintsN &aConstraints,
1.255 + const MediaEnginePrefs &aPrefs);
1.256 +
1.257 + void GuessCapability(const VideoTrackConstraintsN &aConstraints,
1.258 + const MediaEnginePrefs &aPrefs);
1.259 +};
1.260 +
1.261 +class MediaEngineWebRTCAudioSource : public MediaEngineAudioSource,
1.262 + public webrtc::VoEMediaProcess
1.263 +{
1.264 +public:
1.265 + MediaEngineWebRTCAudioSource(webrtc::VoiceEngine* aVoiceEnginePtr, int aIndex,
1.266 + const char* name, const char* uuid)
1.267 + : mVoiceEngine(aVoiceEnginePtr)
1.268 + , mMonitor("WebRTCMic.Monitor")
1.269 + , mCapIndex(aIndex)
1.270 + , mChannel(-1)
1.271 + , mInitDone(false)
1.272 + , mStarted(false)
1.273 + , mSamples(0)
1.274 + , mEchoOn(false), mAgcOn(false), mNoiseOn(false)
1.275 + , mEchoCancel(webrtc::kEcDefault)
1.276 + , mAGC(webrtc::kAgcDefault)
1.277 + , mNoiseSuppress(webrtc::kNsDefault)
1.278 + , mPlayoutDelay(0)
1.279 + , mNullTransport(nullptr) {
1.280 + MOZ_ASSERT(aVoiceEnginePtr);
1.281 + mState = kReleased;
1.282 + mDeviceName.Assign(NS_ConvertUTF8toUTF16(name));
1.283 + mDeviceUUID.Assign(NS_ConvertUTF8toUTF16(uuid));
1.284 + Init();
1.285 + }
1.286 + ~MediaEngineWebRTCAudioSource() { Shutdown(); }
1.287 +
1.288 + virtual void GetName(nsAString&);
1.289 + virtual void GetUUID(nsAString&);
1.290 +
1.291 + virtual nsresult Allocate(const AudioTrackConstraintsN &aConstraints,
1.292 + const MediaEnginePrefs &aPrefs);
1.293 + virtual nsresult Deallocate();
1.294 + virtual nsresult Start(SourceMediaStream*, TrackID);
1.295 + virtual nsresult Stop(SourceMediaStream*, TrackID);
1.296 + virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
1.297 + virtual nsresult Config(bool aEchoOn, uint32_t aEcho,
1.298 + bool aAgcOn, uint32_t aAGC,
1.299 + bool aNoiseOn, uint32_t aNoise,
1.300 + int32_t aPlayoutDelay);
1.301 +
1.302 + virtual void NotifyPull(MediaStreamGraph* aGraph,
1.303 + SourceMediaStream *aSource,
1.304 + TrackID aId,
1.305 + StreamTime aDesiredTime,
1.306 + TrackTicks &aLastEndTime);
1.307 +
1.308 + virtual bool IsFake() {
1.309 + return false;
1.310 + }
1.311 +
1.312 + // VoEMediaProcess.
1.313 + void Process(int channel, webrtc::ProcessingTypes type,
1.314 + int16_t audio10ms[], int length,
1.315 + int samplingFreq, bool isStereo);
1.316 +
1.317 + NS_DECL_THREADSAFE_ISUPPORTS
1.318 +
1.319 +private:
1.320 + static const unsigned int KMaxDeviceNameLength = 128;
1.321 + static const unsigned int KMaxUniqueIdLength = 256;
1.322 +
1.323 + void Init();
1.324 + void Shutdown();
1.325 +
1.326 + webrtc::VoiceEngine* mVoiceEngine;
1.327 + ScopedCustomReleasePtr<webrtc::VoEBase> mVoEBase;
1.328 + ScopedCustomReleasePtr<webrtc::VoEExternalMedia> mVoERender;
1.329 + ScopedCustomReleasePtr<webrtc::VoENetwork> mVoENetwork;
1.330 + ScopedCustomReleasePtr<webrtc::VoEAudioProcessing> mVoEProcessing;
1.331 + ScopedCustomReleasePtr<webrtc::VoECallReport> mVoECallReport;
1.332 +
1.333 + // mMonitor protects mSources[] access/changes, and transitions of mState
1.334 + // from kStarted to kStopped (which are combined with EndTrack()).
1.335 + // mSources[] is accessed from webrtc threads.
1.336 + Monitor mMonitor;
1.337 + nsTArray<SourceMediaStream *> mSources; // When this goes empty, we shut down HW
1.338 +
1.339 + int mCapIndex;
1.340 + int mChannel;
1.341 + TrackID mTrackID;
1.342 + bool mInitDone;
1.343 + bool mStarted;
1.344 + int mSamples; // int to avoid conversions when comparing/etc to samplingFreq & length
1.345 +
1.346 + nsString mDeviceName;
1.347 + nsString mDeviceUUID;
1.348 +
1.349 + bool mEchoOn, mAgcOn, mNoiseOn;
1.350 + webrtc::EcModes mEchoCancel;
1.351 + webrtc::AgcModes mAGC;
1.352 + webrtc::NsModes mNoiseSuppress;
1.353 + int32_t mPlayoutDelay;
1.354 +
1.355 + NullTransport *mNullTransport;
1.356 +};
1.357 +
1.358 +class MediaEngineWebRTC : public MediaEngine
1.359 +{
1.360 +public:
1.361 + MediaEngineWebRTC(MediaEnginePrefs &aPrefs);
1.362 +
1.363 + // Clients should ensure to clean-up sources video/audio sources
1.364 + // before invoking Shutdown on this class.
1.365 + void Shutdown();
1.366 +
1.367 + virtual void EnumerateVideoDevices(nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
1.368 + virtual void EnumerateAudioDevices(nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
1.369 +
1.370 +private:
1.371 + ~MediaEngineWebRTC() {
1.372 + Shutdown();
1.373 +#ifdef MOZ_B2G_CAMERA
1.374 + AsyncLatencyLogger::Get()->Release();
1.375 +#endif
1.376 + // XXX
1.377 + gFarendObserver = nullptr;
1.378 + }
1.379 +
1.380 + Mutex mMutex;
1.381 + // protected with mMutex:
1.382 +
1.383 + webrtc::VideoEngine* mVideoEngine;
1.384 + webrtc::VoiceEngine* mVoiceEngine;
1.385 +
1.386 + // Need this to avoid unneccesary WebRTC calls while enumerating.
1.387 + bool mVideoEngineInit;
1.388 + bool mAudioEngineInit;
1.389 + bool mHasTabVideoSource;
1.390 +
1.391 + // Store devices we've already seen in a hashtable for quick return.
1.392 + // Maps UUID to MediaEngineSource (one set for audio, one for video).
1.393 + nsRefPtrHashtable<nsStringHashKey, MediaEngineWebRTCVideoSource > mVideoSources;
1.394 + nsRefPtrHashtable<nsStringHashKey, MediaEngineWebRTCAudioSource > mAudioSources;
1.395 +};
1.396 +
1.397 +}
1.398 +
1.399 +#endif /* NSMEDIAENGINEWEBRTC_H_ */
