michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef _PEER_CONNECTION_MEDIA_H_
michael@0: #define _PEER_CONNECTION_MEDIA_H_
michael@0: 
michael@0: #include <string>
michael@0: #include <vector>
michael@0: #include <map>
michael@0: 
michael@0: #include "nspr.h"
michael@0: #include "prlock.h"
michael@0: 
michael@0: #include "mozilla/RefPtr.h"
michael@0: #include "nsComponentManagerUtils.h"
michael@0: 
michael@0: #ifdef USE_FAKE_MEDIA_STREAMS
michael@0: #include "FakeMediaStreams.h"
michael@0: #else
michael@0: #include "DOMMediaStream.h"
michael@0: #include "MediaSegment.h"
michael@0: #endif
michael@0: 
michael@0: #include "AudioSegment.h"
michael@0: 
michael@0: #ifdef MOZILLA_INTERNAL_API
michael@0: #include "Layers.h"
michael@0: #include "VideoUtils.h"
michael@0: #include "ImageLayers.h"
michael@0: #include "VideoSegment.h"
michael@0: #endif
michael@0: 
michael@0: namespace mozilla {
michael@0: class DataChannel;
michael@0: namespace dom {
michael@0: class RTCInboundRTPStreamStats;
michael@0: class RTCOutboundRTPStreamStats;
michael@0: }
michael@0: }
michael@0: 
michael@0: #include "nricectx.h"
michael@0: #include "nriceresolver.h"
michael@0: #include "nricemediastream.h"
michael@0: #include "MediaPipeline.h"
michael@0: 
michael@0: namespace sipcc {
michael@0: 
michael@0: class PeerConnectionImpl;
michael@0: class PeerConnectionMedia;
michael@0: 
michael@0: /* Temporary for providing audio data */
michael@0: class Fake_AudioGenerator {
michael@0:  public:
michael@0:   typedef mozilla::DOMMediaStream DOMMediaStream;
michael@0: 
michael@0: Fake_AudioGenerator(DOMMediaStream* aStream) : mStream(aStream), mCount(0) {
michael@0:     mTimer = do_CreateInstance("@mozilla.org/timer;1");
michael@0:     MOZ_ASSERT(mTimer);
michael@0: 
michael@0:     // Make a track
michael@0:     mozilla::AudioSegment *segment = new mozilla::AudioSegment();
michael@0:     mStream->GetStream()->AsSourceStream()->AddTrack(1, 16000, 0, segment);
michael@0: 
michael@0:     // Set the timer
michael@0:     mTimer->InitWithFuncCallback(Callback, this, 100, nsITimer::TYPE_REPEATING_PRECISE);
michael@0:   }
michael@0: 
michael@0:   static void Callback(nsITimer* timer, void *arg) {
michael@0:     Fake_AudioGenerator* gen = static_cast<Fake_AudioGenerator*>(arg);
michael@0: 
michael@0:     nsRefPtr<mozilla::SharedBuffer> samples = mozilla::SharedBuffer::Create(1600 * sizeof(int16_t));
michael@0:     int16_t* data = static_cast<int16_t*>(samples->Data());
michael@0:     for (int i=0; i<1600; i++) {
michael@0:       data[i] = ((gen->mCount % 8) * 4000) - (7*4000)/2;
michael@0:       ++gen->mCount;
michael@0:     }
michael@0: 
michael@0:     mozilla::AudioSegment segment;
michael@0:     nsAutoTArray<const int16_t*,1> channelData;
michael@0:     channelData.AppendElement(data);
michael@0:     segment.AppendFrames(samples.forget(), channelData, 1600);
michael@0:     gen->mStream->GetStream()->AsSourceStream()->AppendToTrack(1, &segment);
michael@0:   }
michael@0: 
michael@0:  private:
michael@0:   nsCOMPtr<nsITimer> mTimer;
michael@0:   nsRefPtr<DOMMediaStream> mStream;
michael@0:   int mCount;
michael@0: };
michael@0: 
michael@0: /* Temporary for providing video data */
michael@0: #ifdef MOZILLA_INTERNAL_API
michael@0: class Fake_VideoGenerator {
michael@0:  public:
michael@0:   typedef mozilla::DOMMediaStream DOMMediaStream;
michael@0:   typedef mozilla::gfx::IntSize IntSize;
michael@0: 
michael@0:   Fake_VideoGenerator(DOMMediaStream* aStream) {
michael@0:     mStream = aStream;
michael@0:     mCount = 0;
michael@0:     mTimer = do_CreateInstance("@mozilla.org/timer;1");
michael@0:     MOZ_ASSERT(mTimer);
michael@0: 
michael@0:     // Make a track
michael@0:     mozilla::VideoSegment *segment = new mozilla::VideoSegment();
michael@0:     mStream->GetStream()->AsSourceStream()->AddTrack(1, mozilla::USECS_PER_S, 0, segment);
michael@0:     mStream->GetStream()->AsSourceStream()->AdvanceKnownTracksTime(mozilla::STREAM_TIME_MAX);
michael@0: 
michael@0:     // Set the timer. Set to 10 fps.
michael@0:     mTimer->InitWithFuncCallback(Callback, this, 100, nsITimer::TYPE_REPEATING_SLACK);
michael@0:   }
michael@0: 
michael@0:   static void Callback(nsITimer* timer, void *arg) {
michael@0:     Fake_VideoGenerator* gen = static_cast<Fake_VideoGenerator*>(arg);
michael@0: 
michael@0:     const uint32_t WIDTH = 640;
michael@0:     const uint32_t HEIGHT = 480;
michael@0: 
michael@0:     // Allocate a single blank Image
michael@0:     nsRefPtr<mozilla::layers::ImageContainer> container =
michael@0:       mozilla::layers::LayerManager::CreateImageContainer();
michael@0: 
michael@0:     nsRefPtr<mozilla::layers::Image> image =
michael@0:       container->CreateImage(mozilla::ImageFormat::PLANAR_YCBCR);
michael@0: 
michael@0:     int len = ((WIDTH * HEIGHT) * 3 / 2);
michael@0:     mozilla::layers::PlanarYCbCrImage* planar =
michael@0:       static_cast<mozilla::layers::PlanarYCbCrImage*>(image.get());
michael@0:     uint8_t* frame = (uint8_t*) PR_Malloc(len);
michael@0:     ++gen->mCount;
michael@0:     memset(frame, (gen->mCount / 8) & 0xff, len); // Rotating colors
michael@0: 
michael@0:     const uint8_t lumaBpp = 8;
michael@0:     const uint8_t chromaBpp = 4;
michael@0: 
michael@0:     mozilla::layers::PlanarYCbCrData data;
michael@0:     data.mYChannel = frame;
michael@0:     data.mYSize = IntSize(WIDTH, HEIGHT);
michael@0:     data.mYStride = (int32_t) (WIDTH * lumaBpp / 8.0);
michael@0:     data.mCbCrStride = (int32_t) (WIDTH * chromaBpp / 8.0);
michael@0:     data.mCbChannel = frame + HEIGHT * data.mYStride;
michael@0:     data.mCrChannel = data.mCbChannel + HEIGHT * data.mCbCrStride / 2;
michael@0:     data.mCbCrSize = IntSize(WIDTH / 2, HEIGHT / 2);
michael@0:     data.mPicX = 0;
michael@0:     data.mPicY = 0;
michael@0:     data.mPicSize = IntSize(WIDTH, HEIGHT);
michael@0:     data.mStereoMode = mozilla::StereoMode::MONO;
michael@0: 
michael@0:     // SetData copies data, so we can free the frame
michael@0:     planar->SetData(data);
michael@0:     PR_Free(frame);
michael@0: 
michael@0:     // AddTrack takes ownership of segment
michael@0:     mozilla::VideoSegment *segment = new mozilla::VideoSegment();
michael@0:     // 10 fps.
michael@0:     segment->AppendFrame(image.forget(), mozilla::USECS_PER_S / 10,
michael@0:                          IntSize(WIDTH, HEIGHT));
michael@0: 
michael@0:     gen->mStream->GetStream()->AsSourceStream()->AppendToTrack(1, segment);
michael@0:   }
michael@0: 
michael@0:  private:
michael@0:   nsCOMPtr<nsITimer> mTimer;
michael@0:   nsRefPtr<DOMMediaStream> mStream;
michael@0:   int mCount;
michael@0: };
michael@0: #endif
michael@0: 
michael@0: 
michael@0: class SourceStreamInfo {
michael@0: public:
michael@0:   typedef mozilla::DOMMediaStream DOMMediaStream;
michael@0: 
michael@0:   SourceStreamInfo(DOMMediaStream* aMediaStream,
michael@0:                    PeerConnectionMedia *aParent)
michael@0:       : mMediaStream(aMediaStream),
michael@0:         mParent(aParent) {
michael@0:     MOZ_ASSERT(mMediaStream);
michael@0:   }
michael@0: 
michael@0:   SourceStreamInfo(already_AddRefed<DOMMediaStream>& aMediaStream,
michael@0:                   PeerConnectionMedia *aParent)
michael@0:       : mMediaStream(aMediaStream),
michael@0:         mParent(aParent) {
michael@0:     MOZ_ASSERT(mMediaStream);
michael@0:   }
michael@0: 
michael@0:   // This method exists for stats and the unittests.
michael@0:   // It allows visibility into the pipelines and flows.
michael@0:   const std::map<mozilla::TrackID, mozilla::RefPtr<mozilla::MediaPipeline>>&
michael@0:   GetPipelines() const { return mPipelines; }
michael@0:   mozilla::RefPtr<mozilla::MediaPipeline> GetPipelineByLevel_m(int level);
michael@0: 
michael@0: protected:
michael@0:   std::map<mozilla::TrackID, mozilla::RefPtr<mozilla::MediaPipeline>> mPipelines;
michael@0:   nsRefPtr<DOMMediaStream> mMediaStream;
michael@0:   PeerConnectionMedia *mParent;
michael@0: };
michael@0: 
michael@0: // TODO(ekr@rtfm.com): Refactor {Local,Remote}SourceStreamInfo
michael@0: // bug 837539.
michael@0: class LocalSourceStreamInfo : public SourceStreamInfo {
michael@0: public:
michael@0:   typedef mozilla::DOMMediaStream DOMMediaStream;
michael@0: 
michael@0:   LocalSourceStreamInfo(DOMMediaStream *aMediaStream,
michael@0:                         PeerConnectionMedia *aParent)
michael@0:       : SourceStreamInfo(aMediaStream, aParent) {}
michael@0: 
michael@0:   ~LocalSourceStreamInfo() {
michael@0:     mMediaStream = nullptr;
michael@0:   }
michael@0: 
michael@0:   DOMMediaStream* GetMediaStream() {
michael@0:     return mMediaStream;
michael@0:   }
michael@0:   void StorePipeline(int aTrack, mozilla::RefPtr<mozilla::MediaPipeline> aPipeline);
michael@0: 
michael@0:   void ExpectAudio(const mozilla::TrackID);
michael@0:   void ExpectVideo(const mozilla::TrackID);
michael@0:   unsigned AudioTrackCount();
michael@0:   unsigned VideoTrackCount();
michael@0:   void DetachTransport_s();
michael@0:   void DetachMedia_m();
michael@0: 
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(LocalSourceStreamInfo)
michael@0: private:
michael@0:   nsTArray<mozilla::TrackID> mAudioTracks;
michael@0:   nsTArray<mozilla::TrackID> mVideoTracks;
michael@0: };
michael@0: 
michael@0: class RemoteSourceStreamInfo : public SourceStreamInfo {
michael@0:  public:
michael@0:   typedef mozilla::DOMMediaStream DOMMediaStream;
michael@0: 
michael@0:   RemoteSourceStreamInfo(already_AddRefed<DOMMediaStream> aMediaStream,
michael@0:                          PeerConnectionMedia *aParent)
michael@0:     : SourceStreamInfo(aMediaStream, aParent),
michael@0:       mTrackTypeHints(0) {}
michael@0: 
michael@0:   DOMMediaStream* GetMediaStream() {
michael@0:     return mMediaStream;
michael@0:   }
michael@0:   void StorePipeline(int aTrack, bool aIsVideo,
michael@0:                      mozilla::RefPtr<mozilla::MediaPipeline> aPipeline);
michael@0: 
michael@0:   bool SetUsingBundle_m(int aLevel, bool decision);
michael@0: 
michael@0:   void DetachTransport_s();
michael@0:   void DetachMedia_m();
michael@0: 
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteSourceStreamInfo)
michael@0: 
michael@0: public:
michael@0:   DOMMediaStream::TrackTypeHints mTrackTypeHints;
michael@0:  private:
michael@0:   std::map<int, bool> mTypes;
michael@0: };
michael@0: 
michael@0: class PeerConnectionMedia : public sigslot::has_slots<> {
michael@0:  public:
michael@0:   PeerConnectionMedia(PeerConnectionImpl *parent);
michael@0:   ~PeerConnectionMedia() {}
michael@0: 
michael@0:   nsresult Init(const std::vector<mozilla::NrIceStunServer>& stun_servers,
michael@0:                 const std::vector<mozilla::NrIceTurnServer>& turn_servers);
michael@0:   // WARNING: This destroys the object!
michael@0:   void SelfDestruct();
michael@0: 
michael@0:   mozilla::RefPtr<mozilla::NrIceCtx> ice_ctx() const { return mIceCtx; }
michael@0: 
michael@0:   mozilla::RefPtr<mozilla::NrIceMediaStream> ice_media_stream(size_t i) const {
michael@0:     // TODO(ekr@rtfm.com): If someone asks for a value that doesn't exist,
michael@0:     // make one.
michael@0:     if (i >= mIceStreams.size()) {
michael@0:       return nullptr;
michael@0:     }
michael@0:     return mIceStreams[i];
michael@0:   }
michael@0: 
michael@0:   size_t num_ice_media_streams() const {
michael@0:     return mIceStreams.size();
michael@0:   }
michael@0: 
michael@0:   // Add a stream
michael@0:   nsresult AddStream(nsIDOMMediaStream* aMediaStream, uint32_t *stream_id);
michael@0: 
michael@0:   // Remove a stream
michael@0:   nsresult RemoveStream(nsIDOMMediaStream* aMediaStream, uint32_t *stream_id);
michael@0: 
michael@0:   // Get a specific local stream
michael@0:   uint32_t LocalStreamsLength()
michael@0:   {
michael@0:     return mLocalSourceStreams.Length();
michael@0:   }
michael@0:   LocalSourceStreamInfo* GetLocalStream(int index);
michael@0: 
michael@0:   // Get a specific remote stream
michael@0:   uint32_t RemoteStreamsLength()
michael@0:   {
michael@0:     return mRemoteSourceStreams.Length();
michael@0:   }
michael@0:   RemoteSourceStreamInfo* GetRemoteStream(int index);
michael@0: 
michael@0:   bool SetUsingBundle_m(int level, bool decision);
michael@0:   bool UpdateFilterFromRemoteDescription_m(
michael@0:       int level,
michael@0:       nsAutoPtr<mozilla::MediaPipelineFilter> filter);
michael@0: 
michael@0:   // Add a remote stream. Returns the index in index
michael@0:   nsresult AddRemoteStream(nsRefPtr<RemoteSourceStreamInfo> aInfo, int *aIndex);
michael@0:   nsresult AddRemoteStreamHint(int aIndex, bool aIsVideo);
michael@0: 
michael@0:   const nsCOMPtr<nsIThread>& GetMainThread() const { return mMainThread; }
michael@0:   const nsCOMPtr<nsIEventTarget>& GetSTSThread() const { return mSTSThread; }
michael@0: 
michael@0:   // Get a transport flow either RTP/RTCP for a particular stream
michael@0:   // A stream can be of audio/video/datachannel/budled(?) types
michael@0:   mozilla::RefPtr<mozilla::TransportFlow> GetTransportFlow(int aStreamIndex,
michael@0:                                                            bool aIsRtcp) {
michael@0:     int index_inner = aStreamIndex * 2 + (aIsRtcp ? 1 : 0);
michael@0: 
michael@0:     if (mTransportFlows.find(index_inner) == mTransportFlows.end())
michael@0:       return nullptr;
michael@0: 
michael@0:     return mTransportFlows[index_inner];
michael@0:   }
michael@0: 
michael@0:   // Add a transport flow
michael@0:   void AddTransportFlow(int aIndex, bool aRtcp,
michael@0:                         const mozilla::RefPtr<mozilla::TransportFlow> &aFlow) {
michael@0:     int index_inner = aIndex * 2 + (aRtcp ? 1 : 0);
michael@0: 
michael@0:     MOZ_ASSERT(!mTransportFlows[index_inner]);
michael@0:     mTransportFlows[index_inner] = aFlow;
michael@0:   }
michael@0: 
michael@0:   mozilla::RefPtr<mozilla::MediaSessionConduit> GetConduit(int aStreamIndex, bool aReceive) {
michael@0:     int index_inner = aStreamIndex * 2 + (aReceive ? 0 : 1);
michael@0: 
michael@0:     if (mConduits.find(index_inner) == mConduits.end())
michael@0:       return nullptr;
michael@0: 
michael@0:     return mConduits[index_inner];
michael@0:   }
michael@0: 
michael@0:   // Add a conduit
michael@0:   void AddConduit(int aIndex, bool aReceive,
michael@0:                   const mozilla::RefPtr<mozilla::MediaSessionConduit> &aConduit) {
michael@0:     int index_inner = aIndex * 2 + (aReceive ? 0 : 1);
michael@0: 
michael@0:     MOZ_ASSERT(!mConduits[index_inner]);
michael@0:     mConduits[index_inner] = aConduit;
michael@0:   }
michael@0: 
michael@0:   // ICE state signals
michael@0:   sigslot::signal2<mozilla::NrIceCtx*, mozilla::NrIceCtx::GatheringState>
michael@0:       SignalIceGatheringStateChange;
michael@0:   sigslot::signal2<mozilla::NrIceCtx*, mozilla::NrIceCtx::ConnectionState>
michael@0:       SignalIceConnectionStateChange;
michael@0: 
michael@0:  private:
michael@0:   // Shutdown media transport. Must be called on STS thread.
michael@0:   void ShutdownMediaTransport_s();
michael@0: 
michael@0:   // Final destruction of the media stream. Must be called on the main
michael@0:   // thread.
michael@0:   void SelfDestruct_m();
michael@0: 
michael@0:   // ICE events
michael@0:   void IceGatheringStateChange_s(mozilla::NrIceCtx* ctx,
michael@0:                                  mozilla::NrIceCtx::GatheringState state);
michael@0:   void IceConnectionStateChange_s(mozilla::NrIceCtx* ctx,
michael@0:                                   mozilla::NrIceCtx::ConnectionState state);
michael@0:   void IceStreamReady(mozilla::NrIceMediaStream *aStream);
michael@0: 
michael@0:   void IceGatheringStateChange_m(mozilla::NrIceCtx* ctx,
michael@0:                                  mozilla::NrIceCtx::GatheringState state);
michael@0:   void IceConnectionStateChange_m(mozilla::NrIceCtx* ctx,
michael@0:                                   mozilla::NrIceCtx::ConnectionState state);
michael@0: 
michael@0:   // The parent PC
michael@0:   PeerConnectionImpl *mParent;
michael@0: 
michael@0:   // A list of streams returned from GetUserMedia
michael@0:   mozilla::Mutex mLocalSourceStreamsLock;
michael@0:   nsTArray<nsRefPtr<LocalSourceStreamInfo> > mLocalSourceStreams;
michael@0: 
michael@0:   // A list of streams provided by the other side
michael@0:   nsTArray<nsRefPtr<RemoteSourceStreamInfo> > mRemoteSourceStreams;
michael@0: 
michael@0:   // ICE objects
michael@0:   mozilla::RefPtr<mozilla::NrIceCtx> mIceCtx;
michael@0:   std::vector<mozilla::RefPtr<mozilla::NrIceMediaStream> > mIceStreams;
michael@0: 
michael@0:   // DNS
michael@0:   nsRefPtr<mozilla::NrIceResolver> mDNSResolver;
michael@0: 
michael@0:   // Transport flows: even is RTP, odd is RTCP
michael@0:   std::map<int, mozilla::RefPtr<mozilla::TransportFlow> > mTransportFlows;
michael@0: 
michael@0:   // Conduits: even is receive, odd is transmit (for easier correlation with
michael@0:   // flows)
michael@0:   std::map<int, mozilla::RefPtr<mozilla::MediaSessionConduit> > mConduits;
michael@0: 
michael@0:   // The main thread.
michael@0:   nsCOMPtr<nsIThread> mMainThread;
michael@0: 
michael@0:   // The STS thread.
michael@0:   nsCOMPtr<nsIEventTarget> mSTSThread;
michael@0: 
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(PeerConnectionMedia)
michael@0: };
michael@0: 
michael@0: }  // namespace sipcc
michael@0: #endif