The Tor Browser / file comparison
comparison: media/webrtc/signaling/src/media-conduit/MediaConduitInterface.h
media/webrtc/signaling/src/media-conduit/MediaConduitInterface.h
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:6c211242f88f |
|---|---|
| 1 /* This Source Code Form is subject to the terms of the Mozilla Public | |
| 2 * License, v. 2.0. If a copy of the MPL was not distributed with this file, | |
| 3 * You can obtain one at http://mozilla.org/MPL/2.0/. */ | |
| 4 | |
| 5 #ifndef MEDIA_CONDUIT_ABSTRACTION_ | |
| 6 #define MEDIA_CONDUIT_ABSTRACTION_ | |
| 7 | |
| 8 #include "nsISupportsImpl.h" | |
| 9 #include "nsXPCOM.h" | |
| 10 #include "nsDOMNavigationTiming.h" | |
| 11 #include "mozilla/RefPtr.h" | |
| 12 #include "CodecConfig.h" | |
| 13 #include "VideoTypes.h" | |
| 14 #include "MediaConduitErrors.h" | |
| 15 | |
| 16 #include "ImageContainer.h" | |
| 17 | |
| 18 #include <vector> | |
| 19 | |
| 20 namespace mozilla { | |
| 21 /** | |
| 22 * Abstract Interface for transporting RTP packets - audio/vidoeo | |
| 23 * The consumers of this interface are responsible for passing in | |
| 24 * the RTPfied media packets | |
| 25 */ | |
| 26 class TransportInterface | |
| 27 { | |
| 28 public: | |
| 29 virtual ~TransportInterface() {} | |
| 30 | |
| 31 /** | |
| 32 * RTP Transport Function to be implemented by concrete transport implementation | |
| 33 * @param data : RTP Packet (audio/video) to be transported | |
| 34 * @param len : Length of the media packet | |
| 35 * @result : NS_OK on success, NS_ERROR_FAILURE otherwise | |
| 36 */ | |
| 37 virtual nsresult SendRtpPacket(const void* data, int len) = 0; | |
| 38 | |
| 39 /** | |
| 40 * RTCP Transport Function to be implemented by concrete transport implementation | |
| 41 * @param data : RTCP Packet to be transported | |
| 42 * @param len : Length of the RTCP packet | |
| 43 * @result : NS_OK on success, NS_ERROR_FAILURE otherwise | |
| 44 */ | |
| 45 virtual nsresult SendRtcpPacket(const void* data, int len) = 0; | |
| 46 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TransportInterface) | |
| 47 }; | |
| 48 | |
| 49 /** | |
| 50 * This class wraps image object for VideoRenderer::RenderVideoFrame() | |
| 51 * callback implementation to use for rendering. | |
| 52 */ | |
| 53 class ImageHandle | |
| 54 { | |
| 55 public: | |
| 56 ImageHandle(layers::Image* image) : mImage(image) {} | |
| 57 | |
| 58 const RefPtr<layers::Image>& GetImage() const { return mImage; } | |
| 59 | |
| 60 private: | |
| 61 RefPtr<layers::Image> mImage; | |
| 62 }; | |
| 63 | |
| 64 /** | |
| 65 * 1. Abstract renderer for video data | |
| 66 * 2. This class acts as abstract interface between the video-engine and | |
| 67 * video-engine agnostic renderer implementation. | |
| 68 * 3. Concrete implementation of this interface is responsible for | |
| 69 * processing and/or rendering the obtained raw video frame to appropriate | |
| 70 * output , say, <video> | |
| 71 */ | |
| 72 class VideoRenderer | |
| 73 { | |
| 74 public: | |
| 75 virtual ~VideoRenderer() {} | |
| 76 | |
| 77 /** | |
| 78 * Callback Function reportng any change in the video-frame dimensions | |
| 79 * @param width: current width of the video @ decoder | |
| 80 * @param height: current height of the video @ decoder | |
| 81 * @param number_of_streams: number of participating video streams | |
| 82 */ | |
| 83 virtual void FrameSizeChange(unsigned int width, | |
| 84 unsigned int height, | |
| 85 unsigned int number_of_streams) = 0; | |
| 86 | |
| 87 /** | |
| 88 * Callback Function reporting decoded I420 frame for processing. | |
| 89 * @param buffer: pointer to decoded video frame | |
| 90 * @param buffer_size: size of the decoded frame | |
| 91 * @param time_stamp: Decoder timestamp, typically 90KHz as per RTP | |
| 92 * @render_time: Wall-clock time at the decoder for synchronization | |
| 93 * purposes in milliseconds | |
| 94 * @handle: opaque handle for image object of decoded video frame. | |
| 95 * NOTE: If decoded video frame is passed through buffer , it is the | |
| 96 * responsibility of the concrete implementations of this class to own copy | |
| 97 * of the frame if needed for time longer than scope of this callback. | |
| 98 * Such implementations should be quick in processing the frames and return | |
| 99 * immediately. | |
| 100 * On the other hand, if decoded video frame is passed through handle, the | |
| 101 * implementations should keep a reference to the (ref-counted) image object | |
| 102 * inside until it's no longer needed. | |
| 103 */ | |
| 104 virtual void RenderVideoFrame(const unsigned char* buffer, | |
| 105 unsigned int buffer_size, | |
| 106 uint32_t time_stamp, | |
| 107 int64_t render_time, | |
| 108 const ImageHandle& handle) = 0; | |
| 109 | |
| 110 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoRenderer) | |
| 111 }; | |
| 112 | |
| 113 | |
| 114 /** | |
| 115 * Generic Interface for representing Audio/Video Session | |
| 116 * MediaSession conduit is identified by 2 main components | |
| 117 * 1. Attached Transport Interface for inbound and outbound RTP transport | |
| 118 * 2. Attached Renderer Interface for rendering media data off the network | |
| 119 * This class hides specifics of Media-Engine implementation from the consumers | |
| 120 * of this interface. | |
| 121 * Also provides codec configuration API for the media sent and recevied | |
| 122 */ | |
| 123 class MediaSessionConduit | |
| 124 { | |
| 125 public: | |
| 126 enum Type { AUDIO, VIDEO } ; | |
| 127 | |
| 128 virtual ~MediaSessionConduit() {} | |
| 129 | |
| 130 virtual Type type() const = 0; | |
| 131 | |
| 132 /** | |
| 133 * Function triggered on Incoming RTP packet from the remote | |
| 134 * endpoint by the transport implementation. | |
| 135 * @param data : RTP Packet (audio/video) to be processed | |
| 136 * @param len : Length of the media packet | |
| 137 * Obtained packets are passed to the Media-Engine for further | |
| 138 * processing , say, decoding | |
| 139 */ | |
| 140 virtual MediaConduitErrorCode ReceivedRTPPacket(const void *data, int len) = 0; | |
| 141 | |
| 142 /** | |
| 143 * Function triggered on Incoming RTCP packet from the remote | |
| 144 * endpoint by the transport implementation. | |
| 145 * @param data : RTCP Packet (audio/video) to be processed | |
| 146 * @param len : Length of the media packet | |
| 147 * Obtained packets are passed to the Media-Engine for further | |
| 148 * processing , say, decoding | |
| 149 */ | |
| 150 virtual MediaConduitErrorCode ReceivedRTCPPacket(const void *data, int len) = 0; | |
| 151 | |
| 152 | |
| 153 /** | |
| 154 * Function to attach Transport end-point of the Media conduit. | |
| 155 * @param aTransport: Reference to the concrete teansport implementation | |
| 156 * Note: Multiple invocations of this call , replaces existing transport with | |
| 157 * with the new one. | |
| 158 */ | |
| 159 virtual MediaConduitErrorCode AttachTransport(RefPtr<TransportInterface> aTransport) = 0; | |
| 160 | |
| 161 virtual bool GetLocalSSRC(unsigned int* ssrc) = 0; | |
| 162 virtual bool GetRemoteSSRC(unsigned int* ssrc) = 0; | |
| 163 | |
| 164 /** | |
| 165 * Functions returning stats needed by w3c stats model. | |
| 166 */ | |
| 167 virtual bool GetAVStats(int32_t* jitterBufferDelayMs, | |
| 168 int32_t* playoutBufferDelayMs, | |
| 169 int32_t* avSyncOffsetMs) = 0; | |
| 170 virtual bool GetRTPStats(unsigned int* jitterMs, | |
| 171 unsigned int* cumulativeLost) = 0; | |
| 172 virtual bool GetRTCPReceiverReport(DOMHighResTimeStamp* timestamp, | |
| 173 uint32_t* jitterMs, | |
| 174 uint32_t* packetsReceived, | |
| 175 uint64_t* bytesReceived, | |
| 176 uint32_t* cumulativeLost, | |
| 177 int32_t* rttMs) = 0; | |
| 178 virtual bool GetRTCPSenderReport(DOMHighResTimeStamp* timestamp, | |
| 179 unsigned int* packetsSent, | |
| 180 uint64_t* bytesSent) = 0; | |
| 181 | |
| 182 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSessionConduit) | |
| 183 | |
| 184 }; | |
| 185 | |
| 186 // Abstract base classes for external encoder/decoder. | |
| 187 class VideoEncoder | |
| 188 { | |
| 189 public: | |
| 190 virtual ~VideoEncoder() {}; | |
| 191 }; | |
| 192 | |
| 193 class VideoDecoder | |
| 194 { | |
| 195 public: | |
| 196 virtual ~VideoDecoder() {}; | |
| 197 }; | |
| 198 | |
| 199 /** | |
| 200 * MediaSessionConduit for video | |
| 201 * Refer to the comments on MediaSessionConduit above for overall | |
| 202 * information | |
| 203 */ | |
| 204 class VideoSessionConduit : public MediaSessionConduit | |
| 205 { | |
| 206 public: | |
| 207 /** | |
| 208 * Factory function to create and initialize a Video Conduit Session | |
| 209 * return: Concrete VideoSessionConduitObject or nullptr in the case | |
| 210 * of failure | |
| 211 */ | |
| 212 static RefPtr<VideoSessionConduit> Create(VideoSessionConduit *aOther); | |
| 213 | |
| 214 enum FrameRequestType | |
| 215 { | |
| 216 FrameRequestNone, | |
| 217 FrameRequestFir, | |
| 218 FrameRequestPli, | |
| 219 FrameRequestUnknown | |
| 220 }; | |
| 221 | |
| 222 VideoSessionConduit() : mFrameRequestMethod(FrameRequestNone), | |
| 223 mUsingNackBasic(false) {} | |
| 224 | |
| 225 virtual ~VideoSessionConduit() {} | |
| 226 | |
| 227 virtual Type type() const { return VIDEO; } | |
| 228 | |
| 229 /** | |
| 230 * Function to attach Renderer end-point of the Media-Video conduit. | |
| 231 * @param aRenderer : Reference to the concrete Video renderer implementation | |
| 232 * Note: Multiple invocations of this API shall remove an existing renderer | |
| 233 * and attaches the new to the Conduit. | |
| 234 */ | |
| 235 virtual MediaConduitErrorCode AttachRenderer(RefPtr<VideoRenderer> aRenderer) = 0; | |
| 236 virtual void DetachRenderer() = 0; | |
| 237 | |
| 238 /** | |
| 239 * Function to deliver a capture video frame for encoding and transport | |
| 240 * @param video_frame: pointer to captured video-frame. | |
| 241 * @param video_frame_length: size of the frame | |
| 242 * @param width, height: dimensions of the frame | |
| 243 * @param video_type: Type of the video frame - I420, RAW | |
| 244 * @param captured_time: timestamp when the frame was captured. | |
| 245 * if 0 timestamp is automatcally generated | |
| 246 * NOTE: ConfigureSendMediaCodec() MUST be called before this function can be invoked | |
| 247 * This ensures the inserted video-frames can be transmitted by the conduit | |
| 248 */ | |
| 249 virtual MediaConduitErrorCode SendVideoFrame(unsigned char* video_frame, | |
| 250 unsigned int video_frame_length, | |
| 251 unsigned short width, | |
| 252 unsigned short height, | |
| 253 VideoType video_type, | |
| 254 uint64_t capture_time) = 0; | |
| 255 | |
| 256 /** | |
| 257 * Function to configure send codec for the video session | |
| 258 * @param sendSessionConfig: CodecConfiguration | |
| 259 * @result: On Success, the video engine is configured with passed in codec for send | |
| 260 * On failure, video engine transmit functionality is disabled. | |
| 261 * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting | |
| 262 * transmission sub-system on the engine | |
| 263 * | |
| 264 */ | |
| 265 virtual MediaConduitErrorCode ConfigureSendMediaCodec(const VideoCodecConfig* sendSessionConfig) = 0; | |
| 266 | |
| 267 /** | |
| 268 * Function to configurelist of receive codecs for the video session | |
| 269 * @param sendSessionConfig: CodecConfiguration | |
| 270 * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting | |
| 271 * reception sub-system on the engine | |
| 272 * | |
| 273 */ | |
| 274 virtual MediaConduitErrorCode ConfigureRecvMediaCodecs( | |
| 275 const std::vector<VideoCodecConfig* >& recvCodecConfigList) = 0; | |
| 276 | |
| 277 /** | |
| 278 * Set an external encoder | |
| 279 * @param encoder | |
| 280 * @result: on success, we will use the specified encoder | |
| 281 */ | |
| 282 virtual MediaConduitErrorCode SetExternalSendCodec(int pltype, | |
| 283 VideoEncoder* encoder) = 0; | |
| 284 | |
| 285 /** | |
| 286 * Set an external decoder | |
| 287 * @param decoder | |
| 288 * @result: on success, we will use the specified decoder | |
| 289 */ | |
| 290 virtual MediaConduitErrorCode SetExternalRecvCodec(int pltype, | |
| 291 VideoDecoder* decoder) = 0; | |
| 292 | |
| 293 /** | |
| 294 * These methods allow unit tests to double-check that the | |
| 295 * max-fs and max-fr related settings are as expected. | |
| 296 */ | |
| 297 virtual unsigned short SendingWidth() = 0; | |
| 298 | |
| 299 virtual unsigned short SendingHeight() = 0; | |
| 300 | |
| 301 virtual unsigned int SendingMaxFs() = 0; | |
| 302 | |
| 303 virtual unsigned int SendingMaxFr() = 0; | |
| 304 | |
| 305 /** | |
| 306 * These methods allow unit tests to double-check that the | |
| 307 * rtcp-fb settings are as expected. | |
| 308 */ | |
| 309 FrameRequestType FrameRequestMethod() const { | |
| 310 return mFrameRequestMethod; | |
| 311 } | |
| 312 | |
| 313 bool UsingNackBasic() const { | |
| 314 return mUsingNackBasic; | |
| 315 } | |
| 316 | |
| 317 protected: | |
| 318 /* RTCP feedback settings, for unit testing purposes */ | |
| 319 FrameRequestType mFrameRequestMethod; | |
| 320 bool mUsingNackBasic; | |
| 321 }; | |
| 322 | |
| 323 /** | |
| 324 * MediaSessionConduit for audio | |
| 325 * Refer to the comments on MediaSessionConduit above for overall | |
| 326 * information | |
| 327 */ | |
| 328 class AudioSessionConduit : public MediaSessionConduit | |
| 329 { | |
| 330 public: | |
| 331 | |
| 332 /** | |
| 333 * Factory function to create and initialize an Audio Conduit Session | |
| 334 * return: Concrete AudioSessionConduitObject or nullptr in the case | |
| 335 * of failure | |
| 336 */ | |
| 337 static mozilla::RefPtr<AudioSessionConduit> Create(AudioSessionConduit *aOther); | |
| 338 | |
| 339 virtual ~AudioSessionConduit() {} | |
| 340 | |
| 341 virtual Type type() const { return AUDIO; } | |
| 342 | |
| 343 | |
| 344 /** | |
| 345 * Function to deliver externally captured audio sample for encoding and transport | |
| 346 * @param audioData [in]: Pointer to array containing a frame of audio | |
| 347 * @param lengthSamples [in]: Length of audio frame in samples in multiple of 10 milliseconds | |
| 348 * Ex: Frame length is 160, 320, 440 for 16, 32, 44 kHz sampling rates | |
| 349 respectively. | |
| 350 audioData[] is lengthSamples in size | |
| 351 say, for 16kz sampling rate, audioData[] should contain 160 | |
| 352 samples of 16-bits each for a 10m audio frame. | |
| 353 * @param samplingFreqHz [in]: Frequency/rate of the sampling in Hz ( 16000, 32000 ...) | |
| 354 * @param capture_delay [in]: Approx Delay from recording until it is delivered to VoiceEngine | |
| 355 in milliseconds. | |
| 356 * NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can be invoked | |
| 357 * This ensures the inserted audio-samples can be transmitted by the conduit | |
| 358 * | |
| 359 */ | |
| 360 virtual MediaConduitErrorCode SendAudioFrame(const int16_t audioData[], | |
| 361 int32_t lengthSamples, | |
| 362 int32_t samplingFreqHz, | |
| 363 int32_t capture_delay) = 0; | |
| 364 | |
| 365 /** | |
| 366 * Function to grab a decoded audio-sample from the media engine for rendering | |
| 367 * / playoutof length 10 milliseconds. | |
| 368 * | |
| 369 * @param speechData [in]: Pointer to a array to which a 10ms frame of audio will be copied | |
| 370 * @param samplingFreqHz [in]: Frequency of the sampling for playback in Hertz (16000, 32000,..) | |
| 371 * @param capture_delay [in]: Estimated Time between reading of the samples to rendering/playback | |
| 372 * @param lengthSamples [out]: Will contain length of the audio frame in samples at return. | |
| 373 Ex: A value of 160 implies 160 samples each of 16-bits was copied | |
| 374 into speechData | |
| 375 * NOTE: This function should be invoked every 10 milliseconds for the best | |
| 376 * peformance | |
| 377 * NOTE: ConfigureRecvMediaCodec() SHOULD be called before this function can be invoked | |
| 378 * This ensures the decoded samples are ready for reading. | |
| 379 * | |
| 380 */ | |
| 381 virtual MediaConduitErrorCode GetAudioFrame(int16_t speechData[], | |
| 382 int32_t samplingFreqHz, | |
| 383 int32_t capture_delay, | |
| 384 int& lengthSamples) = 0; | |
| 385 | |
| 386 /** | |
| 387 * Function to configure send codec for the audio session | |
| 388 * @param sendSessionConfig: CodecConfiguration | |
| 389 * NOTE: See VideoConduit for more information | |
| 390 */ | |
| 391 | |
| 392 virtual MediaConduitErrorCode ConfigureSendMediaCodec(const AudioCodecConfig* sendCodecConfig) = 0; | |
| 393 | |
| 394 /** | |
| 395 * Function to configure list of receive codecs for the audio session | |
| 396 * @param sendSessionConfig: CodecConfiguration | |
| 397 * NOTE: See VideoConduit for more information | |
| 398 */ | |
| 399 virtual MediaConduitErrorCode ConfigureRecvMediaCodecs( | |
| 400 const std::vector<AudioCodecConfig* >& recvCodecConfigList) = 0; | |
| 401 /** | |
| 402 * Function to enable the audio level extension | |
| 403 * @param enabled: enable extension | |
| 404 * @param id: id to be used for this rtp header extension | |
| 405 * NOTE: See AudioConduit for more information | |
| 406 */ | |
| 407 virtual MediaConduitErrorCode EnableAudioLevelExtension(bool enabled, uint8_t id) = 0; | |
| 408 | |
| 409 }; | |
| 410 } | |
| 411 #endif |
