michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef MEDIA_CONDUIT_ABSTRACTION_
michael@0: #define MEDIA_CONDUIT_ABSTRACTION_
michael@0: 
michael@0: #include "nsISupportsImpl.h"
michael@0: #include "nsXPCOM.h"
michael@0: #include "nsDOMNavigationTiming.h"
michael@0: #include "mozilla/RefPtr.h"
michael@0: #include "CodecConfig.h"
michael@0: #include "VideoTypes.h"
michael@0: #include "MediaConduitErrors.h"
michael@0: 
michael@0: #include "ImageContainer.h"
michael@0: 
michael@0: #include <vector>
michael@0: 
michael@0: namespace mozilla {
michael@0: /**
michael@0:  * Abstract Interface for transporting RTP packets - audio/vidoeo
michael@0:  * The consumers of this interface are responsible for passing in
michael@0:  * the RTPfied media packets
michael@0:  */
michael@0: class TransportInterface
michael@0: {
michael@0: public:
michael@0:   virtual ~TransportInterface() {}
michael@0: 
michael@0:   /**
michael@0:    * RTP Transport Function to be implemented by concrete transport implementation
michael@0:    * @param data : RTP Packet (audio/video) to be transported
michael@0:    * @param len  : Length of the media packet
michael@0:    * @result     : NS_OK on success, NS_ERROR_FAILURE otherwise
michael@0:    */
michael@0:   virtual nsresult SendRtpPacket(const void* data, int len) = 0;
michael@0: 
michael@0:   /**
michael@0:    * RTCP Transport Function to be implemented by concrete transport implementation
michael@0:    * @param data : RTCP Packet to be transported
michael@0:    * @param len  : Length of the RTCP packet
michael@0:    * @result     : NS_OK on success, NS_ERROR_FAILURE otherwise
michael@0:    */
michael@0:   virtual nsresult SendRtcpPacket(const void* data, int len) = 0;
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TransportInterface)
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * This class wraps image object for VideoRenderer::RenderVideoFrame()
michael@0:  * callback implementation to use for rendering.
michael@0:  */
michael@0: class ImageHandle
michael@0: {
michael@0: public:
michael@0:   ImageHandle(layers::Image* image) : mImage(image) {}
michael@0: 
michael@0:   const RefPtr<layers::Image>& GetImage() const { return mImage; }
michael@0: 
michael@0: private:
michael@0:   RefPtr<layers::Image> mImage;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * 1. Abstract renderer for video data
michael@0:  * 2. This class acts as abstract interface between the video-engine and
michael@0:  *    video-engine agnostic renderer implementation.
michael@0:  * 3. Concrete implementation of this interface is responsible for
michael@0:  *    processing and/or rendering the obtained raw video frame to appropriate
michael@0:  *    output , say, <video>
michael@0:  */
michael@0: class VideoRenderer
michael@0: {
michael@0:  public:
michael@0:   virtual ~VideoRenderer() {}
michael@0: 
michael@0:   /**
michael@0:    * Callback Function reportng any change in the video-frame dimensions
michael@0:    * @param width:  current width of the video @ decoder
michael@0:    * @param height: current height of the video @ decoder
michael@0:    * @param number_of_streams: number of participating video streams
michael@0:    */
michael@0:   virtual void FrameSizeChange(unsigned int width,
michael@0:                                unsigned int height,
michael@0:                                unsigned int number_of_streams) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Callback Function reporting decoded I420 frame for processing.
michael@0:    * @param buffer: pointer to decoded video frame
michael@0:    * @param buffer_size: size of the decoded frame
michael@0:    * @param time_stamp: Decoder timestamp, typically 90KHz as per RTP
michael@0:    * @render_time: Wall-clock time at the decoder for synchronization
michael@0:    *                purposes in milliseconds
michael@0:    * @handle: opaque handle for image object of decoded video frame.
michael@0:    * NOTE: If decoded video frame is passed through buffer , it is the
michael@0:    * responsibility of the concrete implementations of this class to own copy
michael@0:    * of the frame if needed for time longer than scope of this callback.
michael@0:    * Such implementations should be quick in processing the frames and return
michael@0:    * immediately.
michael@0:    * On the other hand, if decoded video frame is passed through handle, the
michael@0:    * implementations should keep a reference to the (ref-counted) image object
michael@0:    * inside until it's no longer needed.
michael@0:    */
michael@0:   virtual void RenderVideoFrame(const unsigned char* buffer,
michael@0:                                 unsigned int buffer_size,
michael@0:                                 uint32_t time_stamp,
michael@0:                                 int64_t render_time,
michael@0:                                 const ImageHandle& handle) = 0;
michael@0: 
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoRenderer)
michael@0: };
michael@0: 
michael@0: 
michael@0: /**
michael@0:  * Generic Interface for representing Audio/Video Session
michael@0:  * MediaSession conduit is identified by 2 main components
michael@0:  * 1. Attached Transport Interface for inbound and outbound RTP transport
michael@0:  * 2. Attached Renderer Interface for rendering media data off the network
michael@0:  * This class hides specifics of Media-Engine implementation from the consumers
michael@0:  * of this interface.
michael@0:  * Also provides codec configuration API for the media sent and recevied
michael@0:  */
michael@0: class MediaSessionConduit
michael@0: {
michael@0: public:
michael@0:   enum Type { AUDIO, VIDEO } ;
michael@0: 
michael@0:   virtual ~MediaSessionConduit() {}
michael@0: 
michael@0:   virtual Type type() const = 0;
michael@0: 
michael@0:   /**
michael@0:    * Function triggered on Incoming RTP packet from the remote
michael@0:    * endpoint by the transport implementation.
michael@0:    * @param data : RTP Packet (audio/video) to be processed
michael@0:    * @param len  : Length of the media packet
michael@0:    * Obtained packets are passed to the Media-Engine for further
michael@0:    * processing , say, decoding
michael@0:    */
michael@0:   virtual MediaConduitErrorCode ReceivedRTPPacket(const void *data, int len) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Function triggered on Incoming RTCP packet from the remote
michael@0:    * endpoint by the transport implementation.
michael@0:    * @param data : RTCP Packet (audio/video) to be processed
michael@0:    * @param len  : Length of the media packet
michael@0:    * Obtained packets are passed to the Media-Engine for further
michael@0:    * processing , say, decoding
michael@0:    */
michael@0:   virtual MediaConduitErrorCode ReceivedRTCPPacket(const void *data, int len) = 0;
michael@0: 
michael@0: 
michael@0:   /**
michael@0:    * Function to attach Transport end-point of the Media conduit.
michael@0:    * @param aTransport: Reference to the concrete teansport implementation
michael@0:    * Note: Multiple invocations of this call , replaces existing transport with
michael@0:    * with the new one.
michael@0:    */
michael@0:   virtual MediaConduitErrorCode AttachTransport(RefPtr<TransportInterface> aTransport) = 0;
michael@0: 
michael@0:   virtual bool GetLocalSSRC(unsigned int* ssrc) = 0;
michael@0:   virtual bool GetRemoteSSRC(unsigned int* ssrc) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Functions returning stats needed by w3c stats model.
michael@0:    */
michael@0:   virtual bool GetAVStats(int32_t* jitterBufferDelayMs,
michael@0:                           int32_t* playoutBufferDelayMs,
michael@0:                           int32_t* avSyncOffsetMs) = 0;
michael@0:   virtual bool GetRTPStats(unsigned int* jitterMs,
michael@0:                            unsigned int* cumulativeLost) = 0;
michael@0:   virtual bool GetRTCPReceiverReport(DOMHighResTimeStamp* timestamp,
michael@0:                                      uint32_t* jitterMs,
michael@0:                                      uint32_t* packetsReceived,
michael@0:                                      uint64_t* bytesReceived,
michael@0:                                      uint32_t* cumulativeLost,
michael@0:                                      int32_t* rttMs) = 0;
michael@0:   virtual bool GetRTCPSenderReport(DOMHighResTimeStamp* timestamp,
michael@0:                                    unsigned int* packetsSent,
michael@0:                                    uint64_t* bytesSent) = 0;
michael@0: 
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSessionConduit)
michael@0: 
michael@0: };
michael@0: 
michael@0: // Abstract base classes for external encoder/decoder.
michael@0: class VideoEncoder
michael@0: {
michael@0: public:
michael@0:   virtual ~VideoEncoder() {};
michael@0: };
michael@0: 
michael@0: class VideoDecoder
michael@0: {
michael@0: public:
michael@0:   virtual ~VideoDecoder() {};
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * MediaSessionConduit for video
michael@0:  * Refer to the comments on MediaSessionConduit above for overall
michael@0:  * information
michael@0:  */
michael@0: class VideoSessionConduit : public MediaSessionConduit
michael@0: {
michael@0: public:
michael@0:   /**
michael@0:    * Factory function to create and initialize a Video Conduit Session
michael@0:    * return: Concrete VideoSessionConduitObject or nullptr in the case
michael@0:    *         of failure
michael@0:    */
michael@0:   static RefPtr<VideoSessionConduit> Create(VideoSessionConduit *aOther);
michael@0: 
michael@0:   enum FrameRequestType
michael@0:   {
michael@0:     FrameRequestNone,
michael@0:     FrameRequestFir,
michael@0:     FrameRequestPli,
michael@0:     FrameRequestUnknown
michael@0:   };
michael@0: 
michael@0:   VideoSessionConduit() : mFrameRequestMethod(FrameRequestNone),
michael@0:                           mUsingNackBasic(false) {}
michael@0: 
michael@0:   virtual ~VideoSessionConduit() {}
michael@0: 
michael@0:   virtual Type type() const { return VIDEO; }
michael@0: 
michael@0:   /**
michael@0:    * Function to attach Renderer end-point of the Media-Video conduit.
michael@0:    * @param aRenderer : Reference to the concrete Video renderer implementation
michael@0:    * Note: Multiple invocations of this API shall remove an existing renderer
michael@0:    * and attaches the new to the Conduit.
michael@0:    */
michael@0:   virtual MediaConduitErrorCode AttachRenderer(RefPtr<VideoRenderer> aRenderer) = 0;
michael@0:   virtual void DetachRenderer() = 0;
michael@0: 
michael@0:   /**
michael@0:    * Function to deliver a capture video frame for encoding and transport
michael@0:    * @param video_frame: pointer to captured video-frame.
michael@0:    * @param video_frame_length: size of the frame
michael@0:    * @param width, height: dimensions of the frame
michael@0:    * @param video_type: Type of the video frame - I420, RAW
michael@0:    * @param captured_time: timestamp when the frame was captured.
michael@0:    *                       if 0 timestamp is automatcally generated
michael@0:    * NOTE: ConfigureSendMediaCodec() MUST be called before this function can be invoked
michael@0:    *       This ensures the inserted video-frames can be transmitted by the conduit
michael@0:    */
michael@0:   virtual MediaConduitErrorCode SendVideoFrame(unsigned char* video_frame,
michael@0:                                                unsigned int video_frame_length,
michael@0:                                                unsigned short width,
michael@0:                                                unsigned short height,
michael@0:                                                VideoType video_type,
michael@0:                                                uint64_t capture_time) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Function to configure send codec for the video session
michael@0:    * @param sendSessionConfig: CodecConfiguration
michael@0:    * @result: On Success, the video engine is configured with passed in codec for send
michael@0:    *          On failure, video engine transmit functionality is disabled.
michael@0:    * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
michael@0:    *        transmission sub-system on the engine
michael@0:    *
michael@0:    */
michael@0:   virtual MediaConduitErrorCode ConfigureSendMediaCodec(const VideoCodecConfig* sendSessionConfig) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Function to configurelist of receive codecs for the video session
michael@0:    * @param sendSessionConfig: CodecConfiguration
michael@0:    * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
michael@0:    *        reception sub-system on the engine
michael@0:    *
michael@0:    */
michael@0:   virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
michael@0:                                 const std::vector<VideoCodecConfig* >& recvCodecConfigList) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Set an external encoder
michael@0:    * @param encoder
michael@0:    * @result: on success, we will use the specified encoder
michael@0:    */
michael@0:   virtual MediaConduitErrorCode SetExternalSendCodec(int pltype,
michael@0:                                                      VideoEncoder* encoder) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Set an external decoder
michael@0:    * @param decoder
michael@0:    * @result: on success, we will use the specified decoder
michael@0:    */
michael@0:   virtual MediaConduitErrorCode SetExternalRecvCodec(int pltype,
michael@0:                                                      VideoDecoder* decoder) = 0;
michael@0: 
michael@0:   /**
michael@0:    * These methods allow unit tests to double-check that the
michael@0:    * max-fs and max-fr related settings are as expected.
michael@0:    */
michael@0:   virtual unsigned short SendingWidth() = 0;
michael@0: 
michael@0:   virtual unsigned short SendingHeight() = 0;
michael@0: 
michael@0:   virtual unsigned int SendingMaxFs() = 0;
michael@0: 
michael@0:   virtual unsigned int SendingMaxFr() = 0;
michael@0: 
michael@0:   /**
michael@0:     * These methods allow unit tests to double-check that the
michael@0:     * rtcp-fb settings are as expected.
michael@0:     */
michael@0:     FrameRequestType FrameRequestMethod() const {
michael@0:       return mFrameRequestMethod;
michael@0:     }
michael@0: 
michael@0:     bool UsingNackBasic() const {
michael@0:       return mUsingNackBasic;
michael@0:     }
michael@0: 
michael@0:    protected:
michael@0:      /* RTCP feedback settings, for unit testing purposes */
michael@0:      FrameRequestType mFrameRequestMethod;
michael@0:      bool mUsingNackBasic;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * MediaSessionConduit for audio
michael@0:  * Refer to the comments on MediaSessionConduit above for overall
michael@0:  * information
michael@0:  */
michael@0: class AudioSessionConduit : public MediaSessionConduit
michael@0: {
michael@0: public:
michael@0: 
michael@0:    /**
michael@0:     * Factory function to create and initialize an Audio Conduit Session
michael@0:     * return: Concrete AudioSessionConduitObject or nullptr in the case
michael@0:     *         of failure
michael@0:     */
michael@0:   static mozilla::RefPtr<AudioSessionConduit> Create(AudioSessionConduit *aOther);
michael@0: 
michael@0:   virtual ~AudioSessionConduit() {}
michael@0: 
michael@0:   virtual Type type() const { return AUDIO; }
michael@0: 
michael@0: 
michael@0:   /**
michael@0:    * Function to deliver externally captured audio sample for encoding and transport
michael@0:    * @param audioData [in]: Pointer to array containing a frame of audio
michael@0:    * @param lengthSamples [in]: Length of audio frame in samples in multiple of 10 milliseconds
michael@0:   *                             Ex: Frame length is 160, 320, 440 for 16, 32, 44 kHz sampling rates
michael@0:                                     respectively.
michael@0:                                     audioData[] is lengthSamples in size
michael@0:                                     say, for 16kz sampling rate, audioData[] should contain 160
michael@0:                                     samples of 16-bits each for a 10m audio frame.
michael@0:    * @param samplingFreqHz [in]: Frequency/rate of the sampling in Hz ( 16000, 32000 ...)
michael@0:    * @param capture_delay [in]:  Approx Delay from recording until it is delivered to VoiceEngine
michael@0:                                  in milliseconds.
michael@0:    * NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can be invoked
michael@0:    *       This ensures the inserted audio-samples can be transmitted by the conduit
michael@0:    *
michael@0:    */
michael@0:   virtual MediaConduitErrorCode SendAudioFrame(const int16_t audioData[],
michael@0:                                                 int32_t lengthSamples,
michael@0:                                                 int32_t samplingFreqHz,
michael@0:                                                 int32_t capture_delay) = 0;
michael@0: 
michael@0:   /**
michael@0:    * Function to grab a decoded audio-sample from the media engine for rendering
michael@0:    * / playoutof length 10 milliseconds.
michael@0:    *
michael@0:    * @param speechData [in]: Pointer to a array to which a 10ms frame of audio will be copied
michael@0:    * @param samplingFreqHz [in]: Frequency of the sampling for playback in Hertz (16000, 32000,..)
michael@0:    * @param capture_delay [in]: Estimated Time between reading of the samples to rendering/playback
michael@0:    * @param lengthSamples [out]: Will contain length of the audio frame in samples at return.
michael@0:                                  Ex: A value of 160 implies 160 samples each of 16-bits was copied
michael@0:                                      into speechData
michael@0:    * NOTE: This function should be invoked every 10 milliseconds for the best
michael@0:    *          peformance
michael@0:    * NOTE: ConfigureRecvMediaCodec() SHOULD be called before this function can be invoked
michael@0:    *       This ensures the decoded samples are ready for reading.
michael@0:    *
michael@0:    */
michael@0:   virtual MediaConduitErrorCode GetAudioFrame(int16_t speechData[],
michael@0:                                               int32_t samplingFreqHz,
michael@0:                                               int32_t capture_delay,
michael@0:                                               int& lengthSamples) = 0;
michael@0: 
michael@0:    /**
michael@0:     * Function to configure send codec for the audio session
michael@0:     * @param sendSessionConfig: CodecConfiguration
michael@0:     * NOTE: See VideoConduit for more information
michael@0:     */
michael@0: 
michael@0:   virtual MediaConduitErrorCode ConfigureSendMediaCodec(const AudioCodecConfig* sendCodecConfig) = 0;
michael@0: 
michael@0:    /**
michael@0:     * Function to configure list of receive codecs for the audio session
michael@0:     * @param sendSessionConfig: CodecConfiguration
michael@0:     * NOTE: See VideoConduit for more information
michael@0:     */
michael@0:   virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
michael@0:                                 const std::vector<AudioCodecConfig* >& recvCodecConfigList) = 0;
michael@0:    /**
michael@0:     * Function to enable the audio level extension
michael@0:     * @param enabled: enable extension
michael@0:     * @param id: id to be used for this rtp header extension
michael@0:     * NOTE: See AudioConduit for more information
michael@0:     */
michael@0:   virtual MediaConduitErrorCode EnableAudioLevelExtension(bool enabled, uint8_t id) = 0;
michael@0: 
michael@0: };
michael@0: }
michael@0: #endif