1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/media/webrtc/signaling/src/media-conduit/MediaConduitInterface.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,411 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.6 + * You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +#ifndef MEDIA_CONDUIT_ABSTRACTION_
1.9 +#define MEDIA_CONDUIT_ABSTRACTION_
1.10 +
1.11 +#include "nsISupportsImpl.h"
1.12 +#include "nsXPCOM.h"
1.13 +#include "nsDOMNavigationTiming.h"
1.14 +#include "mozilla/RefPtr.h"
1.15 +#include "CodecConfig.h"
1.16 +#include "VideoTypes.h"
1.17 +#include "MediaConduitErrors.h"
1.18 +
1.19 +#include "ImageContainer.h"
1.20 +
1.21 +#include <vector>
1.22 +
1.23 +namespace mozilla {
1.24 +/**
1.25 + * Abstract Interface for transporting RTP packets - audio/vidoeo
1.26 + * The consumers of this interface are responsible for passing in
1.27 + * the RTPfied media packets
1.28 + */
1.29 +class TransportInterface
1.30 +{
1.31 +public:
1.32 + virtual ~TransportInterface() {}
1.33 +
1.34 + /**
1.35 + * RTP Transport Function to be implemented by concrete transport implementation
1.36 + * @param data : RTP Packet (audio/video) to be transported
1.37 + * @param len : Length of the media packet
1.38 + * @result : NS_OK on success, NS_ERROR_FAILURE otherwise
1.39 + */
1.40 + virtual nsresult SendRtpPacket(const void* data, int len) = 0;
1.41 +
1.42 + /**
1.43 + * RTCP Transport Function to be implemented by concrete transport implementation
1.44 + * @param data : RTCP Packet to be transported
1.45 + * @param len : Length of the RTCP packet
1.46 + * @result : NS_OK on success, NS_ERROR_FAILURE otherwise
1.47 + */
1.48 + virtual nsresult SendRtcpPacket(const void* data, int len) = 0;
1.49 + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TransportInterface)
1.50 +};
1.51 +
1.52 +/**
1.53 + * This class wraps image object for VideoRenderer::RenderVideoFrame()
1.54 + * callback implementation to use for rendering.
1.55 + */
1.56 +class ImageHandle
1.57 +{
1.58 +public:
1.59 + ImageHandle(layers::Image* image) : mImage(image) {}
1.60 +
1.61 + const RefPtr<layers::Image>& GetImage() const { return mImage; }
1.62 +
1.63 +private:
1.64 + RefPtr<layers::Image> mImage;
1.65 +};
1.66 +
1.67 +/**
1.68 + * 1. Abstract renderer for video data
1.69 + * 2. This class acts as abstract interface between the video-engine and
1.70 + * video-engine agnostic renderer implementation.
1.71 + * 3. Concrete implementation of this interface is responsible for
1.72 + * processing and/or rendering the obtained raw video frame to appropriate
1.73 + * output , say, <video>
1.74 + */
1.75 +class VideoRenderer
1.76 +{
1.77 + public:
1.78 + virtual ~VideoRenderer() {}
1.79 +
1.80 + /**
1.81 + * Callback Function reportng any change in the video-frame dimensions
1.82 + * @param width: current width of the video @ decoder
1.83 + * @param height: current height of the video @ decoder
1.84 + * @param number_of_streams: number of participating video streams
1.85 + */
1.86 + virtual void FrameSizeChange(unsigned int width,
1.87 + unsigned int height,
1.88 + unsigned int number_of_streams) = 0;
1.89 +
1.90 + /**
1.91 + * Callback Function reporting decoded I420 frame for processing.
1.92 + * @param buffer: pointer to decoded video frame
1.93 + * @param buffer_size: size of the decoded frame
1.94 + * @param time_stamp: Decoder timestamp, typically 90KHz as per RTP
1.95 + * @render_time: Wall-clock time at the decoder for synchronization
1.96 + * purposes in milliseconds
1.97 + * @handle: opaque handle for image object of decoded video frame.
1.98 + * NOTE: If decoded video frame is passed through buffer , it is the
1.99 + * responsibility of the concrete implementations of this class to own copy
1.100 + * of the frame if needed for time longer than scope of this callback.
1.101 + * Such implementations should be quick in processing the frames and return
1.102 + * immediately.
1.103 + * On the other hand, if decoded video frame is passed through handle, the
1.104 + * implementations should keep a reference to the (ref-counted) image object
1.105 + * inside until it's no longer needed.
1.106 + */
1.107 + virtual void RenderVideoFrame(const unsigned char* buffer,
1.108 + unsigned int buffer_size,
1.109 + uint32_t time_stamp,
1.110 + int64_t render_time,
1.111 + const ImageHandle& handle) = 0;
1.112 +
1.113 + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoRenderer)
1.114 +};
1.115 +
1.116 +
1.117 +/**
1.118 + * Generic Interface for representing Audio/Video Session
1.119 + * MediaSession conduit is identified by 2 main components
1.120 + * 1. Attached Transport Interface for inbound and outbound RTP transport
1.121 + * 2. Attached Renderer Interface for rendering media data off the network
1.122 + * This class hides specifics of Media-Engine implementation from the consumers
1.123 + * of this interface.
1.124 + * Also provides codec configuration API for the media sent and recevied
1.125 + */
1.126 +class MediaSessionConduit
1.127 +{
1.128 +public:
1.129 + enum Type { AUDIO, VIDEO } ;
1.130 +
1.131 + virtual ~MediaSessionConduit() {}
1.132 +
1.133 + virtual Type type() const = 0;
1.134 +
1.135 + /**
1.136 + * Function triggered on Incoming RTP packet from the remote
1.137 + * endpoint by the transport implementation.
1.138 + * @param data : RTP Packet (audio/video) to be processed
1.139 + * @param len : Length of the media packet
1.140 + * Obtained packets are passed to the Media-Engine for further
1.141 + * processing , say, decoding
1.142 + */
1.143 + virtual MediaConduitErrorCode ReceivedRTPPacket(const void *data, int len) = 0;
1.144 +
1.145 + /**
1.146 + * Function triggered on Incoming RTCP packet from the remote
1.147 + * endpoint by the transport implementation.
1.148 + * @param data : RTCP Packet (audio/video) to be processed
1.149 + * @param len : Length of the media packet
1.150 + * Obtained packets are passed to the Media-Engine for further
1.151 + * processing , say, decoding
1.152 + */
1.153 + virtual MediaConduitErrorCode ReceivedRTCPPacket(const void *data, int len) = 0;
1.154 +
1.155 +
1.156 + /**
1.157 + * Function to attach Transport end-point of the Media conduit.
1.158 + * @param aTransport: Reference to the concrete teansport implementation
1.159 + * Note: Multiple invocations of this call , replaces existing transport with
1.160 + * with the new one.
1.161 + */
1.162 + virtual MediaConduitErrorCode AttachTransport(RefPtr<TransportInterface> aTransport) = 0;
1.163 +
1.164 + virtual bool GetLocalSSRC(unsigned int* ssrc) = 0;
1.165 + virtual bool GetRemoteSSRC(unsigned int* ssrc) = 0;
1.166 +
1.167 + /**
1.168 + * Functions returning stats needed by w3c stats model.
1.169 + */
1.170 + virtual bool GetAVStats(int32_t* jitterBufferDelayMs,
1.171 + int32_t* playoutBufferDelayMs,
1.172 + int32_t* avSyncOffsetMs) = 0;
1.173 + virtual bool GetRTPStats(unsigned int* jitterMs,
1.174 + unsigned int* cumulativeLost) = 0;
1.175 + virtual bool GetRTCPReceiverReport(DOMHighResTimeStamp* timestamp,
1.176 + uint32_t* jitterMs,
1.177 + uint32_t* packetsReceived,
1.178 + uint64_t* bytesReceived,
1.179 + uint32_t* cumulativeLost,
1.180 + int32_t* rttMs) = 0;
1.181 + virtual bool GetRTCPSenderReport(DOMHighResTimeStamp* timestamp,
1.182 + unsigned int* packetsSent,
1.183 + uint64_t* bytesSent) = 0;
1.184 +
1.185 + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSessionConduit)
1.186 +
1.187 +};
1.188 +
1.189 +// Abstract base classes for external encoder/decoder.
1.190 +class VideoEncoder
1.191 +{
1.192 +public:
1.193 + virtual ~VideoEncoder() {};
1.194 +};
1.195 +
1.196 +class VideoDecoder
1.197 +{
1.198 +public:
1.199 + virtual ~VideoDecoder() {};
1.200 +};
1.201 +
1.202 +/**
1.203 + * MediaSessionConduit for video
1.204 + * Refer to the comments on MediaSessionConduit above for overall
1.205 + * information
1.206 + */
1.207 +class VideoSessionConduit : public MediaSessionConduit
1.208 +{
1.209 +public:
1.210 + /**
1.211 + * Factory function to create and initialize a Video Conduit Session
1.212 + * return: Concrete VideoSessionConduitObject or nullptr in the case
1.213 + * of failure
1.214 + */
1.215 + static RefPtr<VideoSessionConduit> Create(VideoSessionConduit *aOther);
1.216 +
1.217 + enum FrameRequestType
1.218 + {
1.219 + FrameRequestNone,
1.220 + FrameRequestFir,
1.221 + FrameRequestPli,
1.222 + FrameRequestUnknown
1.223 + };
1.224 +
1.225 + VideoSessionConduit() : mFrameRequestMethod(FrameRequestNone),
1.226 + mUsingNackBasic(false) {}
1.227 +
1.228 + virtual ~VideoSessionConduit() {}
1.229 +
1.230 + virtual Type type() const { return VIDEO; }
1.231 +
1.232 + /**
1.233 + * Function to attach Renderer end-point of the Media-Video conduit.
1.234 + * @param aRenderer : Reference to the concrete Video renderer implementation
1.235 + * Note: Multiple invocations of this API shall remove an existing renderer
1.236 + * and attaches the new to the Conduit.
1.237 + */
1.238 + virtual MediaConduitErrorCode AttachRenderer(RefPtr<VideoRenderer> aRenderer) = 0;
1.239 + virtual void DetachRenderer() = 0;
1.240 +
1.241 + /**
1.242 + * Function to deliver a capture video frame for encoding and transport
1.243 + * @param video_frame: pointer to captured video-frame.
1.244 + * @param video_frame_length: size of the frame
1.245 + * @param width, height: dimensions of the frame
1.246 + * @param video_type: Type of the video frame - I420, RAW
1.247 + * @param captured_time: timestamp when the frame was captured.
1.248 + * if 0 timestamp is automatcally generated
1.249 + * NOTE: ConfigureSendMediaCodec() MUST be called before this function can be invoked
1.250 + * This ensures the inserted video-frames can be transmitted by the conduit
1.251 + */
1.252 + virtual MediaConduitErrorCode SendVideoFrame(unsigned char* video_frame,
1.253 + unsigned int video_frame_length,
1.254 + unsigned short width,
1.255 + unsigned short height,
1.256 + VideoType video_type,
1.257 + uint64_t capture_time) = 0;
1.258 +
1.259 + /**
1.260 + * Function to configure send codec for the video session
1.261 + * @param sendSessionConfig: CodecConfiguration
1.262 + * @result: On Success, the video engine is configured with passed in codec for send
1.263 + * On failure, video engine transmit functionality is disabled.
1.264 + * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
1.265 + * transmission sub-system on the engine
1.266 + *
1.267 + */
1.268 + virtual MediaConduitErrorCode ConfigureSendMediaCodec(const VideoCodecConfig* sendSessionConfig) = 0;
1.269 +
1.270 + /**
1.271 + * Function to configurelist of receive codecs for the video session
1.272 + * @param sendSessionConfig: CodecConfiguration
1.273 + * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
1.274 + * reception sub-system on the engine
1.275 + *
1.276 + */
1.277 + virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
1.278 + const std::vector<VideoCodecConfig* >& recvCodecConfigList) = 0;
1.279 +
1.280 + /**
1.281 + * Set an external encoder
1.282 + * @param encoder
1.283 + * @result: on success, we will use the specified encoder
1.284 + */
1.285 + virtual MediaConduitErrorCode SetExternalSendCodec(int pltype,
1.286 + VideoEncoder* encoder) = 0;
1.287 +
1.288 + /**
1.289 + * Set an external decoder
1.290 + * @param decoder
1.291 + * @result: on success, we will use the specified decoder
1.292 + */
1.293 + virtual MediaConduitErrorCode SetExternalRecvCodec(int pltype,
1.294 + VideoDecoder* decoder) = 0;
1.295 +
1.296 + /**
1.297 + * These methods allow unit tests to double-check that the
1.298 + * max-fs and max-fr related settings are as expected.
1.299 + */
1.300 + virtual unsigned short SendingWidth() = 0;
1.301 +
1.302 + virtual unsigned short SendingHeight() = 0;
1.303 +
1.304 + virtual unsigned int SendingMaxFs() = 0;
1.305 +
1.306 + virtual unsigned int SendingMaxFr() = 0;
1.307 +
1.308 + /**
1.309 + * These methods allow unit tests to double-check that the
1.310 + * rtcp-fb settings are as expected.
1.311 + */
1.312 + FrameRequestType FrameRequestMethod() const {
1.313 + return mFrameRequestMethod;
1.314 + }
1.315 +
1.316 + bool UsingNackBasic() const {
1.317 + return mUsingNackBasic;
1.318 + }
1.319 +
1.320 + protected:
1.321 + /* RTCP feedback settings, for unit testing purposes */
1.322 + FrameRequestType mFrameRequestMethod;
1.323 + bool mUsingNackBasic;
1.324 +};
1.325 +
1.326 +/**
1.327 + * MediaSessionConduit for audio
1.328 + * Refer to the comments on MediaSessionConduit above for overall
1.329 + * information
1.330 + */
1.331 +class AudioSessionConduit : public MediaSessionConduit
1.332 +{
1.333 +public:
1.334 +
1.335 + /**
1.336 + * Factory function to create and initialize an Audio Conduit Session
1.337 + * return: Concrete AudioSessionConduitObject or nullptr in the case
1.338 + * of failure
1.339 + */
1.340 + static mozilla::RefPtr<AudioSessionConduit> Create(AudioSessionConduit *aOther);
1.341 +
1.342 + virtual ~AudioSessionConduit() {}
1.343 +
1.344 + virtual Type type() const { return AUDIO; }
1.345 +
1.346 +
1.347 + /**
1.348 + * Function to deliver externally captured audio sample for encoding and transport
1.349 + * @param audioData [in]: Pointer to array containing a frame of audio
1.350 + * @param lengthSamples [in]: Length of audio frame in samples in multiple of 10 milliseconds
1.351 + * Ex: Frame length is 160, 320, 440 for 16, 32, 44 kHz sampling rates
1.352 + respectively.
1.353 + audioData[] is lengthSamples in size
1.354 + say, for 16kz sampling rate, audioData[] should contain 160
1.355 + samples of 16-bits each for a 10m audio frame.
1.356 + * @param samplingFreqHz [in]: Frequency/rate of the sampling in Hz ( 16000, 32000 ...)
1.357 + * @param capture_delay [in]: Approx Delay from recording until it is delivered to VoiceEngine
1.358 + in milliseconds.
1.359 + * NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can be invoked
1.360 + * This ensures the inserted audio-samples can be transmitted by the conduit
1.361 + *
1.362 + */
1.363 + virtual MediaConduitErrorCode SendAudioFrame(const int16_t audioData[],
1.364 + int32_t lengthSamples,
1.365 + int32_t samplingFreqHz,
1.366 + int32_t capture_delay) = 0;
1.367 +
1.368 + /**
1.369 + * Function to grab a decoded audio-sample from the media engine for rendering
1.370 + * / playoutof length 10 milliseconds.
1.371 + *
1.372 + * @param speechData [in]: Pointer to a array to which a 10ms frame of audio will be copied
1.373 + * @param samplingFreqHz [in]: Frequency of the sampling for playback in Hertz (16000, 32000,..)
1.374 + * @param capture_delay [in]: Estimated Time between reading of the samples to rendering/playback
1.375 + * @param lengthSamples [out]: Will contain length of the audio frame in samples at return.
1.376 + Ex: A value of 160 implies 160 samples each of 16-bits was copied
1.377 + into speechData
1.378 + * NOTE: This function should be invoked every 10 milliseconds for the best
1.379 + * peformance
1.380 + * NOTE: ConfigureRecvMediaCodec() SHOULD be called before this function can be invoked
1.381 + * This ensures the decoded samples are ready for reading.
1.382 + *
1.383 + */
1.384 + virtual MediaConduitErrorCode GetAudioFrame(int16_t speechData[],
1.385 + int32_t samplingFreqHz,
1.386 + int32_t capture_delay,
1.387 + int& lengthSamples) = 0;
1.388 +
1.389 + /**
1.390 + * Function to configure send codec for the audio session
1.391 + * @param sendSessionConfig: CodecConfiguration
1.392 + * NOTE: See VideoConduit for more information
1.393 + */
1.394 +
1.395 + virtual MediaConduitErrorCode ConfigureSendMediaCodec(const AudioCodecConfig* sendCodecConfig) = 0;
1.396 +
1.397 + /**
1.398 + * Function to configure list of receive codecs for the audio session
1.399 + * @param sendSessionConfig: CodecConfiguration
1.400 + * NOTE: See VideoConduit for more information
1.401 + */
1.402 + virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
1.403 + const std::vector<AudioCodecConfig* >& recvCodecConfigList) = 0;
1.404 + /**
1.405 + * Function to enable the audio level extension
1.406 + * @param enabled: enable extension
1.407 + * @param id: id to be used for this rtp header extension
1.408 + * NOTE: See AudioConduit for more information
1.409 + */
1.410 + virtual MediaConduitErrorCode EnableAudioLevelExtension(bool enabled, uint8_t id) = 0;
1.411 +
1.412 +};
1.413 +}
1.414 +#endif
