michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef MOZILLA_MEDIASTREAMGRAPH_H_
michael@0: #define MOZILLA_MEDIASTREAMGRAPH_H_
michael@0: 
michael@0: #include "mozilla/Mutex.h"
michael@0: #include "mozilla/LinkedList.h"
michael@0: #include "AudioStream.h"
michael@0: #include "nsTArray.h"
michael@0: #include "nsIRunnable.h"
michael@0: #include "StreamBuffer.h"
michael@0: #include "TimeVarying.h"
michael@0: #include "VideoFrameContainer.h"
michael@0: #include "VideoSegment.h"
michael@0: #include "MainThreadUtils.h"
michael@0: #include "nsAutoRef.h"
michael@0: #include "speex/speex_resampler.h"
michael@0: #include "AudioMixer.h"
michael@0: #include "mozilla/dom/AudioChannelBinding.h"
michael@0: 
michael@0: class nsIRunnable;
michael@0: 
michael@0: template <>
michael@0: class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState>
michael@0: {
michael@0:   public:
michael@0:   static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); }
michael@0: };
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: class DOMMediaStream;
michael@0: 
michael@0: #ifdef PR_LOGGING
michael@0: extern PRLogModuleInfo* gMediaStreamGraphLog;
michael@0: #endif
michael@0: 
michael@0: /**
michael@0:  * Microseconds relative to the start of the graph timeline.
michael@0:  */
michael@0: typedef int64_t GraphTime;
michael@0: const GraphTime GRAPH_TIME_MAX = MEDIA_TIME_MAX;
michael@0: 
michael@0: /*
michael@0:  * MediaStreamGraph is a framework for synchronized audio/video processing
michael@0:  * and playback. It is designed to be used by other browser components such as
michael@0:  * HTML media elements, media capture APIs, real-time media streaming APIs,
michael@0:  * multitrack media APIs, and advanced audio APIs.
michael@0:  *
michael@0:  * The MediaStreamGraph uses a dedicated thread to process media --- the media
michael@0:  * graph thread. This ensures that we can process media through the graph
michael@0:  * without blocking on main-thread activity. The media graph is only modified
michael@0:  * on the media graph thread, to ensure graph changes can be processed without
michael@0:  * interfering with media processing. All interaction with the media graph
michael@0:  * thread is done with message passing.
michael@0:  *
michael@0:  * APIs that modify the graph or its properties are described as "control APIs".
michael@0:  * These APIs are asynchronous; they queue graph changes internally and
michael@0:  * those changes are processed all-at-once by the MediaStreamGraph. The
michael@0:  * MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState
michael@0:  * to ensure that graph changes from a single event loop task are always
michael@0:  * processed all together. Control APIs should only be used on the main thread,
michael@0:  * currently; we may be able to relax that later.
michael@0:  *
michael@0:  * To allow precise synchronization of times in the control API, the
michael@0:  * MediaStreamGraph maintains a "media timeline". Control APIs that take or
michael@0:  * return times use that timeline. Those times never advance during
michael@0:  * an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime().
michael@0:  *
michael@0:  * Media decoding, audio processing and media playback use thread-safe APIs to
michael@0:  * the media graph to ensure they can continue while the main thread is blocked.
michael@0:  *
michael@0:  * When the graph is changed, we may need to throw out buffered data and
michael@0:  * reprocess it. This is triggered automatically by the MediaStreamGraph.
michael@0:  */
michael@0: 
michael@0: class MediaStreamGraph;
michael@0: 
michael@0: /**
michael@0:  * This is a base class for media graph thread listener callbacks.
michael@0:  * Override methods to be notified of audio or video data or changes in stream
michael@0:  * state.
michael@0:  *
michael@0:  * This can be used by stream recorders or network connections that receive
michael@0:  * stream input. It could also be used for debugging.
michael@0:  *
michael@0:  * All notification methods are called from the media graph thread. Overriders
michael@0:  * of these methods are responsible for all synchronization. Beware!
michael@0:  * These methods are called without the media graph monitor held, so
michael@0:  * reentry into media graph methods is possible, although very much discouraged!
michael@0:  * You should do something non-blocking and non-reentrant (e.g. dispatch an
michael@0:  * event to some thread) and return.
michael@0:  * The listener is not allowed to add/remove any listeners from the stream.
michael@0:  *
michael@0:  * When a listener is first attached, we guarantee to send a NotifyBlockingChanged
michael@0:  * callback to notify of the initial blocking state. Also, if a listener is
michael@0:  * attached to a stream that has already finished, we'll call NotifyFinished.
michael@0:  */
michael@0: class MediaStreamListener {
michael@0: protected:
michael@0:   // Protected destructor, to discourage deletion outside of Release():
michael@0:   virtual ~MediaStreamListener() {}
michael@0: 
michael@0: public:
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener)
michael@0: 
michael@0:   enum Consumption {
michael@0:     CONSUMED,
michael@0:     NOT_CONSUMED
michael@0:   };
michael@0:   /**
michael@0:    * Notify that the stream is hooked up and we'd like to start or stop receiving
michael@0:    * data on it. Only fires on SourceMediaStreams.
michael@0:    * The initial state is assumed to be NOT_CONSUMED.
michael@0:    */
michael@0:   virtual void NotifyConsumptionChanged(MediaStreamGraph* aGraph, Consumption aConsuming) {}
michael@0: 
michael@0:   /**
michael@0:    * When a SourceMediaStream has pulling enabled, and the MediaStreamGraph
michael@0:    * control loop is ready to pull, this gets called. A NotifyPull implementation
michael@0:    * is allowed to call the SourceMediaStream methods that alter track
michael@0:    * data. It is not allowed to make other MediaStream API calls, including
michael@0:    * calls to add or remove MediaStreamListeners. It is not allowed to block
michael@0:    * for any length of time.
michael@0:    * aDesiredTime is the stream time we would like to get data up to. Data
michael@0:    * beyond this point will not be played until NotifyPull runs again, so there's
michael@0:    * not much point in providing it. Note that if the stream is blocked for
michael@0:    * some reason, then data before aDesiredTime may not be played immediately.
michael@0:    */
michael@0:   virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) {}
michael@0: 
michael@0:   enum Blocking {
michael@0:     BLOCKED,
michael@0:     UNBLOCKED
michael@0:   };
michael@0:   /**
michael@0:    * Notify that the blocking status of the stream changed. The initial state
michael@0:    * is assumed to be BLOCKED.
michael@0:    */
michael@0:   virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) {}
michael@0: 
michael@0:   /**
michael@0:    * Notify that the stream has data in each track
michael@0:    * for the stream's current time. Once this state becomes true, it will
michael@0:    * always be true since we block stream time from progressing to times where
michael@0:    * there isn't data in each track.
michael@0:    */
michael@0:   virtual void NotifyHasCurrentData(MediaStreamGraph* aGraph) {}
michael@0: 
michael@0:   /**
michael@0:    * Notify that the stream output is advancing. aCurrentTime is the graph's
michael@0:    * current time. MediaStream::GraphTimeToStreamTime can be used to get the
michael@0:    * stream time.
michael@0:    */
michael@0:   virtual void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) {}
michael@0: 
michael@0:   /**
michael@0:    * Notify that the stream finished.
michael@0:    */
michael@0:   virtual void NotifyFinished(MediaStreamGraph* aGraph) {}
michael@0: 
michael@0:   /**
michael@0:    * Notify that your listener has been removed, either due to RemoveListener(),
michael@0:    * or due to the stream being destroyed.  You will get no further notifications.
michael@0:    */
michael@0:   virtual void NotifyRemoved(MediaStreamGraph* aGraph) {}
michael@0: 
michael@0:   enum {
michael@0:     TRACK_EVENT_CREATED = 0x01,
michael@0:     TRACK_EVENT_ENDED = 0x02
michael@0:   };
michael@0:   /**
michael@0:    * Notify that changes to one of the stream tracks have been queued.
michael@0:    * aTrackEvents can be any combination of TRACK_EVENT_CREATED and
michael@0:    * TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
michael@0:    * at aTrackOffset (relative to the start of the stream).
michael@0:    */
michael@0:   virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
michael@0:                                         TrackRate aTrackRate,
michael@0:                                         TrackTicks aTrackOffset,
michael@0:                                         uint32_t aTrackEvents,
michael@0:                                         const MediaSegment& aQueuedMedia) {}
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * This is a base class for media graph thread listener direct callbacks
michael@0:  * from within AppendToTrack().  Note that your regular listener will
michael@0:  * still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
michael@0:  * you must be careful to ignore them if AddDirectListener was successful.
michael@0:  */
michael@0: class MediaStreamDirectListener : public MediaStreamListener
michael@0: {
michael@0: public:
michael@0:   virtual ~MediaStreamDirectListener() {}
michael@0: 
michael@0:   /*
michael@0:    * This will be called on any MediaStreamDirectListener added to a
michael@0:    * a SourceMediaStream when AppendToTrack() is called.  The MediaSegment
michael@0:    * will be the RawSegment (unresampled) if available in AppendToTrack().
michael@0:    * Note that NotifyQueuedTrackChanges() calls will also still occur.
michael@0:    */
michael@0:   virtual void NotifyRealtimeData(MediaStreamGraph* aGraph, TrackID aID,
michael@0:                                   TrackRate aTrackRate,
michael@0:                                   TrackTicks aTrackOffset,
michael@0:                                   uint32_t aTrackEvents,
michael@0:                                   const MediaSegment& aMedia) {}
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * This is a base class for main-thread listener callbacks.
michael@0:  * This callback is invoked on the main thread when the main-thread-visible
michael@0:  * state of a stream has changed.
michael@0:  *
michael@0:  * These methods are called with the media graph monitor held, so
michael@0:  * reentry into general media graph methods is not possible.
michael@0:  * You should do something non-blocking and non-reentrant (e.g. dispatch an
michael@0:  * event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
michael@0:  * would be a good choice.
michael@0:  * The listener is allowed to synchronously remove itself from the stream, but
michael@0:  * not add or remove any other listeners.
michael@0:  */
michael@0: class MainThreadMediaStreamListener {
michael@0: public:
michael@0:   virtual void NotifyMainThreadStateChanged() = 0;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Helper struct used to keep track of memory usage by AudioNodes.
michael@0:  */
michael@0: struct AudioNodeSizes
michael@0: {
michael@0:   size_t mDomNode;
michael@0:   size_t mStream;
michael@0:   size_t mEngine;
michael@0:   nsCString mNodeType;
michael@0: };
michael@0: 
michael@0: class MediaStreamGraphImpl;
michael@0: class SourceMediaStream;
michael@0: class ProcessedMediaStream;
michael@0: class MediaInputPort;
michael@0: class AudioNodeEngine;
michael@0: class AudioNodeExternalInputStream;
michael@0: class AudioNodeStream;
michael@0: struct AudioChunk;
michael@0: 
michael@0: /**
michael@0:  * A stream of synchronized audio and video data. All (not blocked) streams
michael@0:  * progress at the same rate --- "real time". Streams cannot seek. The only
michael@0:  * operation readers can perform on a stream is to read the next data.
michael@0:  *
michael@0:  * Consumers of a stream can be reading from it at different offsets, but that
michael@0:  * should only happen due to the order in which consumers are being run.
michael@0:  * Those offsets must not diverge in the long term, otherwise we would require
michael@0:  * unbounded buffering.
michael@0:  *
michael@0:  * Streams can be in a "blocked" state. While blocked, a stream does not
michael@0:  * produce data. A stream can be explicitly blocked via the control API,
michael@0:  * or implicitly blocked by whatever's generating it (e.g. an underrun in the
michael@0:  * source resource), or implicitly blocked because something consuming it
michael@0:  * blocks, or implicitly because it has finished.
michael@0:  *
michael@0:  * A stream can be in a "finished" state. "Finished" streams are permanently
michael@0:  * blocked.
michael@0:  *
michael@0:  * Transitions into and out of the "blocked" and "finished" states are managed
michael@0:  * by the MediaStreamGraph on the media graph thread.
michael@0:  *
michael@0:  * We buffer media data ahead of the consumers' reading offsets. It is possible
michael@0:  * to have buffered data but still be blocked.
michael@0:  *
michael@0:  * Any stream can have its audio and video playing when requested. The media
michael@0:  * stream graph plays audio by constructing audio output streams as necessary.
michael@0:  * Video is played by setting video frames into an VideoFrameContainer at the right
michael@0:  * time. To ensure video plays in sync with audio, make sure that the same
michael@0:  * stream is playing both the audio and video.
michael@0:  *
michael@0:  * The data in a stream is managed by StreamBuffer. It consists of a set of
michael@0:  * tracks of various types that can start and end over time.
michael@0:  *
michael@0:  * Streams are explicitly managed. The client creates them via
michael@0:  * MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling
michael@0:  * Destroy() when no longer needed (actual destruction will be deferred).
michael@0:  * The actual object is owned by the MediaStreamGraph. The basic idea is that
michael@0:  * main thread objects will keep Streams alive as long as necessary (using the
michael@0:  * cycle collector to clean up whenever needed).
michael@0:  *
michael@0:  * We make them refcounted only so that stream-related messages with MediaStream*
michael@0:  * pointers can be sent to the main thread safely.
michael@0:  *
michael@0:  * The lifetimes of MediaStreams are controlled from the main thread.
michael@0:  * For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM
michael@0:  * wrapper; the DOM wrappers own their associated MediaStreams. When a DOM
michael@0:  * wrapper is destroyed, it sends a Destroy message for the associated
michael@0:  * MediaStream and clears its reference (the last main-thread reference to
michael@0:  * the object). When the Destroy message is processed on the graph
michael@0:  * manager thread we immediately release the affected objects (disentangling them
michael@0:  * from other objects as necessary).
michael@0:  *
michael@0:  * This could cause problems for media processing if a MediaStream is
michael@0:  * destroyed while a downstream MediaStream is still using it. Therefore
michael@0:  * the DOM wrappers must keep upstream MediaStreams alive as long as they
michael@0:  * could be being used in the media graph.
michael@0:  *
michael@0:  * At any time, however, a set of MediaStream wrappers could be
michael@0:  * collected via cycle collection. Destroy messages will be sent
michael@0:  * for those objects in arbitrary order and the MediaStreamGraph has to be able
michael@0:  * to handle this.
michael@0:  */
michael@0: class MediaStream : public mozilla::LinkedListElement<MediaStream> {
michael@0: public:
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
michael@0: 
michael@0:   MediaStream(DOMMediaStream* aWrapper);
michael@0: 
michael@0: protected:
michael@0:   // Protected destructor, to discourage deletion outside of Release():
michael@0:   virtual ~MediaStream()
michael@0:   {
michael@0:     MOZ_COUNT_DTOR(MediaStream);
michael@0:     NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
michael@0:     NS_ASSERTION(mMainThreadListeners.IsEmpty(),
michael@0:                  "All main thread listeners should have been removed");
michael@0:   }
michael@0: 
michael@0: public:
michael@0:   /**
michael@0:    * Returns the graph that owns this stream.
michael@0:    */
michael@0:   MediaStreamGraphImpl* GraphImpl();
michael@0:   MediaStreamGraph* Graph();
michael@0:   /**
michael@0:    * Sets the graph that owns this stream.  Should only be called once.
michael@0:    */
michael@0:   void SetGraphImpl(MediaStreamGraphImpl* aGraph);
michael@0:   void SetGraphImpl(MediaStreamGraph* aGraph);
michael@0: 
michael@0:   // Control API.
michael@0:   // Since a stream can be played multiple ways, we need to combine independent
michael@0:   // volume settings. The aKey parameter is used to keep volume settings
michael@0:   // separate. Since the stream is always playing the same contents, only
michael@0:   // a single audio output stream is used; the volumes are combined.
michael@0:   // Currently only the first enabled audio track is played.
michael@0:   // XXX change this so all enabled audio tracks are mixed and played.
michael@0:   virtual void AddAudioOutput(void* aKey);
michael@0:   virtual void SetAudioOutputVolume(void* aKey, float aVolume);
michael@0:   virtual void RemoveAudioOutput(void* aKey);
michael@0:   // Since a stream can be played multiple ways, we need to be able to
michael@0:   // play to multiple VideoFrameContainers.
michael@0:   // Only the first enabled video track is played.
michael@0:   virtual void AddVideoOutput(VideoFrameContainer* aContainer);
michael@0:   virtual void RemoveVideoOutput(VideoFrameContainer* aContainer);
michael@0:   // Explicitly block. Useful for example if a media element is pausing
michael@0:   // and we need to stop its stream emitting its buffered data.
michael@0:   virtual void ChangeExplicitBlockerCount(int32_t aDelta);
michael@0:   // Events will be dispatched by calling methods of aListener.
michael@0:   virtual void AddListener(MediaStreamListener* aListener);
michael@0:   virtual void RemoveListener(MediaStreamListener* aListener);
michael@0:   // A disabled track has video replaced by black, and audio replaced by
michael@0:   // silence.
michael@0:   void SetTrackEnabled(TrackID aTrackID, bool aEnabled);
michael@0:   // Events will be dispatched by calling methods of aListener. It is the
michael@0:   // responsibility of the caller to remove aListener before it is destroyed.
michael@0:   void AddMainThreadListener(MainThreadMediaStreamListener* aListener)
michael@0:   {
michael@0:     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
michael@0:     mMainThreadListeners.AppendElement(aListener);
michael@0:   }
michael@0:   // It's safe to call this even if aListener is not currently a listener;
michael@0:   // the call will be ignored.
michael@0:   void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener)
michael@0:   {
michael@0:     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
michael@0:     mMainThreadListeners.RemoveElement(aListener);
michael@0:   }
michael@0:   /**
michael@0:    * Ensure a runnable will run on the main thread after running all pending
michael@0:    * updates that were sent from the graph thread or will be sent before the
michael@0:    * graph thread receives the next graph update.
michael@0:    *
michael@0:    * If the graph has been shut down or destroyed, then the runnable will be
michael@0:    * dispatched to the event queue immediately.  If the graph is non-realtime
michael@0:    * and has not started, then the runnable will be run
michael@0:    * synchronously/immediately.  (There are no pending updates in these
michael@0:    * situations.)
michael@0:    *
michael@0:    * Main thread only.
michael@0:    */
michael@0:   void RunAfterPendingUpdates(nsRefPtr<nsIRunnable> aRunnable);
michael@0: 
michael@0:   // Signal that the client is done with this MediaStream. It will be deleted later.
michael@0:   virtual void Destroy();
michael@0:   // Returns the main-thread's view of how much data has been processed by
michael@0:   // this stream.
michael@0:   StreamTime GetCurrentTime()
michael@0:   {
michael@0:     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
michael@0:     return mMainThreadCurrentTime;
michael@0:   }
michael@0:   // Return the main thread's view of whether this stream has finished.
michael@0:   bool IsFinished()
michael@0:   {
michael@0:     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
michael@0:     return mMainThreadFinished;
michael@0:   }
michael@0:   bool IsDestroyed()
michael@0:   {
michael@0:     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
michael@0:     return mMainThreadDestroyed;
michael@0:   }
michael@0: 
michael@0:   friend class MediaStreamGraphImpl;
michael@0:   friend class MediaInputPort;
michael@0:   friend class AudioNodeExternalInputStream;
michael@0: 
michael@0:   virtual SourceMediaStream* AsSourceStream() { return nullptr; }
michael@0:   virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
michael@0:   virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
michael@0: 
michael@0:   // media graph thread only
michael@0:   void Init();
michael@0:   // These Impl methods perform the core functionality of the control methods
michael@0:   // above, on the media graph thread.
michael@0:   /**
michael@0:    * Stop all stream activity and disconnect it from all inputs and outputs.
michael@0:    * This must be idempotent.
michael@0:    */
michael@0:   virtual void DestroyImpl();
michael@0:   StreamTime GetBufferEnd() { return mBuffer.GetEnd(); }
michael@0: #ifdef DEBUG
michael@0:   void DumpTrackInfo() { return mBuffer.DumpTrackInfo(); }
michael@0: #endif
michael@0:   void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
michael@0:   void AddAudioOutputImpl(void* aKey)
michael@0:   {
michael@0:     mAudioOutputs.AppendElement(AudioOutput(aKey));
michael@0:   }
michael@0:   void RemoveAudioOutputImpl(void* aKey);
michael@0:   void AddVideoOutputImpl(already_AddRefed<VideoFrameContainer> aContainer)
michael@0:   {
michael@0:     *mVideoOutputs.AppendElement() = aContainer;
michael@0:   }
michael@0:   void RemoveVideoOutputImpl(VideoFrameContainer* aContainer)
michael@0:   {
michael@0:     mVideoOutputs.RemoveElement(aContainer);
michael@0:   }
michael@0:   void ChangeExplicitBlockerCountImpl(GraphTime aTime, int32_t aDelta)
michael@0:   {
michael@0:     mExplicitBlockerCount.SetAtAndAfter(aTime, mExplicitBlockerCount.GetAt(aTime) + aDelta);
michael@0:   }
michael@0:   void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
michael@0:   void RemoveListenerImpl(MediaStreamListener* aListener);
michael@0:   void RemoveAllListenersImpl();
michael@0:   void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled);
michael@0:   /**
michael@0:    * Returns true when this stream requires the contents of its inputs even if
michael@0:    * its own outputs are not being consumed. This is used to signal inputs to
michael@0:    * this stream that they are being consumed; when they're not being consumed,
michael@0:    * we make some optimizations.
michael@0:    */
michael@0:   virtual bool IsIntrinsicallyConsumed() const
michael@0:   {
michael@0:     return !mAudioOutputs.IsEmpty() || !mVideoOutputs.IsEmpty();
michael@0:   }
michael@0: 
michael@0:   void AddConsumer(MediaInputPort* aPort)
michael@0:   {
michael@0:     mConsumers.AppendElement(aPort);
michael@0:   }
michael@0:   void RemoveConsumer(MediaInputPort* aPort)
michael@0:   {
michael@0:     mConsumers.RemoveElement(aPort);
michael@0:   }
michael@0:   uint32_t ConsumerCount()
michael@0:   {
michael@0:     return mConsumers.Length();
michael@0:   }
michael@0:   const StreamBuffer& GetStreamBuffer() { return mBuffer; }
michael@0:   GraphTime GetStreamBufferStartTime() { return mBufferStartTime; }
michael@0:   /**
michael@0:    * Convert graph time to stream time. aTime must be <= mStateComputedTime
michael@0:    * to ensure we know exactly how much time this stream will be blocked during
michael@0:    * the interval.
michael@0:    */
michael@0:   StreamTime GraphTimeToStreamTime(GraphTime aTime);
michael@0:   /**
michael@0:    * Convert graph time to stream time. aTime can be > mStateComputedTime,
michael@0:    * in which case we optimistically assume the stream will not be blocked
michael@0:    * after mStateComputedTime.
michael@0:    */
michael@0:   StreamTime GraphTimeToStreamTimeOptimistic(GraphTime aTime);
michael@0:   /**
michael@0:    * Convert stream time to graph time. The result can be > mStateComputedTime,
michael@0:    * in which case we did the conversion optimistically assuming the stream
michael@0:    * will not be blocked after mStateComputedTime.
michael@0:    */
michael@0:   GraphTime StreamTimeToGraphTime(StreamTime aTime);
michael@0:   bool IsFinishedOnGraphThread() { return mFinished; }
michael@0:   void FinishOnGraphThread();
michael@0:   /**
michael@0:    * Identify which graph update index we are currently processing.
michael@0:    */
michael@0:   int64_t GetProcessingGraphUpdateIndex();
michael@0: 
michael@0:   bool HasCurrentData() { return mHasCurrentData; }
michael@0: 
michael@0:   StreamBuffer::Track* EnsureTrack(TrackID aTrack, TrackRate aSampleRate);
michael@0: 
michael@0:   void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr);
michael@0: 
michael@0:   DOMMediaStream* GetWrapper()
michael@0:   {
michael@0:     NS_ASSERTION(NS_IsMainThread(), "Only use DOMMediaStream on main thread");
michael@0:     return mWrapper;
michael@0:   }
michael@0: 
michael@0:   // Return true if the main thread needs to observe updates from this stream.
michael@0:   virtual bool MainThreadNeedsUpdates() const
michael@0:   {
michael@0:     return true;
michael@0:   }
michael@0: 
michael@0:   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
michael@0:   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
michael@0: 
michael@0:   void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; }
michael@0: 
michael@0: protected:
michael@0:   virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
michael@0:   {
michael@0:     mBufferStartTime += aBlockedTime;
michael@0:     mGraphUpdateIndices.InsertTimeAtStart(aBlockedTime);
michael@0:     mGraphUpdateIndices.AdvanceCurrentTime(aCurrentTime);
michael@0:     mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime);
michael@0: 
michael@0:     mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime);
michael@0:   }
michael@0: 
michael@0:   // This state is all initialized on the main thread but
michael@0:   // otherwise modified only on the media graph thread.
michael@0: 
michael@0:   // Buffered data. The start of the buffer corresponds to mBufferStartTime.
michael@0:   // Conceptually the buffer contains everything this stream has ever played,
michael@0:   // but we forget some prefix of the buffered data to bound the space usage.
michael@0:   StreamBuffer mBuffer;
michael@0:   // The time when the buffered data could be considered to have started playing.
michael@0:   // This increases over time to account for time the stream was blocked before
michael@0:   // mCurrentTime.
michael@0:   GraphTime mBufferStartTime;
michael@0: 
michael@0:   // Client-set volume of this stream
michael@0:   struct AudioOutput {
michael@0:     AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
michael@0:     void* mKey;
michael@0:     float mVolume;
michael@0:   };
michael@0:   nsTArray<AudioOutput> mAudioOutputs;
michael@0:   nsTArray<nsRefPtr<VideoFrameContainer> > mVideoOutputs;
michael@0:   // We record the last played video frame to avoid redundant setting
michael@0:   // of the current video frame.
michael@0:   VideoFrame mLastPlayedVideoFrame;
michael@0:   // The number of times this stream has been explicitly blocked by the control
michael@0:   // API, minus the number of times it has been explicitly unblocked.
michael@0:   TimeVarying<GraphTime,uint32_t,0> mExplicitBlockerCount;
michael@0:   nsTArray<nsRefPtr<MediaStreamListener> > mListeners;
michael@0:   nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
michael@0:   nsTArray<TrackID> mDisabledTrackIDs;
michael@0: 
michael@0:   // Precomputed blocking status (over GraphTime).
michael@0:   // This is only valid between the graph's mCurrentTime and
michael@0:   // mStateComputedTime. The stream is considered to have
michael@0:   // not been blocked before mCurrentTime (its mBufferStartTime is increased
michael@0:   // as necessary to account for that time instead) --- this avoids us having to
michael@0:   // record the entire history of the stream's blocking-ness in mBlocked.
michael@0:   TimeVarying<GraphTime,bool,5> mBlocked;
michael@0:   // Maps graph time to the graph update that affected this stream at that time
michael@0:   TimeVarying<GraphTime,int64_t,0> mGraphUpdateIndices;
michael@0: 
michael@0:   // MediaInputPorts to which this is connected
michael@0:   nsTArray<MediaInputPort*> mConsumers;
michael@0: 
michael@0:   // Where audio output is going. There is one AudioOutputStream per
michael@0:   // audio track.
michael@0:   struct AudioOutputStream {
michael@0:     // When we started audio playback for this track.
michael@0:     // Add mStream->GetPosition() to find the current audio playback position.
michael@0:     GraphTime mAudioPlaybackStartTime;
michael@0:     // Amount of time that we've wanted to play silence because of the stream
michael@0:     // blocking.
michael@0:     MediaTime mBlockedAudioTime;
michael@0:     // Last tick written to the audio output.
michael@0:     TrackTicks mLastTickWritten;
michael@0:     RefPtr<AudioStream> mStream;
michael@0:     TrackID mTrackID;
michael@0: 
michael@0:     size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
michael@0:     {
michael@0:       size_t amount = 0;
michael@0:       amount += mStream->SizeOfIncludingThis(aMallocSizeOf);
michael@0:       return amount;
michael@0:     }
michael@0:   };
michael@0:   nsTArray<AudioOutputStream> mAudioOutputStreams;
michael@0: 
michael@0:   /**
michael@0:    * When true, this means the stream will be finished once all
michael@0:    * buffered data has been consumed.
michael@0:    */
michael@0:   bool mFinished;
michael@0:   /**
michael@0:    * When true, mFinished is true and we've played all the data in this stream
michael@0:    * and fired NotifyFinished notifications.
michael@0:    */
michael@0:   bool mNotifiedFinished;
michael@0:   /**
michael@0:    * When true, the last NotifyBlockingChanged delivered to the listeners
michael@0:    * indicated that the stream is blocked.
michael@0:    */
michael@0:   bool mNotifiedBlocked;
michael@0:   /**
michael@0:    * True if some data can be present by this stream if/when it's unblocked.
michael@0:    * Set by the stream itself on the MediaStreamGraph thread. Only changes
michael@0:    * from false to true once a stream has data, since we won't
michael@0:    * unblock it until there's more data.
michael@0:    */
michael@0:   bool mHasCurrentData;
michael@0:   /**
michael@0:    * True if mHasCurrentData is true and we've notified listeners.
michael@0:    */
michael@0:   bool mNotifiedHasCurrentData;
michael@0: 
michael@0:   // Temporary data for ordering streams by dependency graph
michael@0:   bool mHasBeenOrdered;
michael@0:   bool mIsOnOrderingStack;
michael@0:   // True if the stream is being consumed (i.e. has track data being played,
michael@0:   // or is feeding into some stream that is being consumed).
michael@0:   bool mIsConsumed;
michael@0:   // Temporary data for computing blocking status of streams
michael@0:   // True if we've added this stream to the set of streams we're computing
michael@0:   // blocking for.
michael@0:   bool mInBlockingSet;
michael@0:   // True if this stream should be blocked in this phase.
michael@0:   bool mBlockInThisPhase;
michael@0: 
michael@0:   // This state is only used on the main thread.
michael@0:   DOMMediaStream* mWrapper;
michael@0:   // Main-thread views of state
michael@0:   StreamTime mMainThreadCurrentTime;
michael@0:   bool mMainThreadFinished;
michael@0:   bool mMainThreadDestroyed;
michael@0: 
michael@0:   // Our media stream graph.  null if destroyed on the graph thread.
michael@0:   MediaStreamGraphImpl* mGraph;
michael@0: 
michael@0:   dom::AudioChannel mAudioChannelType;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * This is a stream into which a decoder can write audio and video.
michael@0:  *
michael@0:  * Audio and video can be written on any thread, but you probably want to
michael@0:  * always write from the same thread to avoid unexpected interleavings.
michael@0:  */
michael@0: class SourceMediaStream : public MediaStream {
michael@0: public:
michael@0:   SourceMediaStream(DOMMediaStream* aWrapper) :
michael@0:     MediaStream(aWrapper),
michael@0:     mLastConsumptionState(MediaStreamListener::NOT_CONSUMED),
michael@0:     mMutex("mozilla::media::SourceMediaStream"),
michael@0:     mUpdateKnownTracksTime(0),
michael@0:     mPullEnabled(false),
michael@0:     mUpdateFinished(false)
michael@0:   {}
michael@0: 
michael@0:   virtual SourceMediaStream* AsSourceStream() { return this; }
michael@0: 
michael@0:   // Media graph thread only
michael@0:   virtual void DestroyImpl();
michael@0: 
michael@0:   // Call these on any thread.
michael@0:   /**
michael@0:    * Enable or disable pulling. When pulling is enabled, NotifyPull
michael@0:    * gets called on MediaStreamListeners for this stream during the
michael@0:    * MediaStreamGraph control loop. Pulling is initially disabled.
michael@0:    * Due to unavoidable race conditions, after a call to SetPullEnabled(false)
michael@0:    * it is still possible for a NotifyPull to occur.
michael@0:    */
michael@0:   void SetPullEnabled(bool aEnabled);
michael@0: 
michael@0:   void AddDirectListener(MediaStreamDirectListener* aListener);
michael@0:   void RemoveDirectListener(MediaStreamDirectListener* aListener);
michael@0: 
michael@0:   /**
michael@0:    * Add a new track to the stream starting at the given base time (which
michael@0:    * must be greater than or equal to the last time passed to
michael@0:    * AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should
michael@0:    * contain data starting after aStart.
michael@0:    */
michael@0:   void AddTrack(TrackID aID, TrackRate aRate, TrackTicks aStart,
michael@0:                 MediaSegment* aSegment);
michael@0: 
michael@0:   /**
michael@0:    * Append media data to a track. Ownership of aSegment remains with the caller,
michael@0:    * but aSegment is emptied.
michael@0:    * Returns false if the data was not appended because no such track exists
michael@0:    * or the stream was already finished.
michael@0:    */
michael@0:   bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
michael@0:   /**
michael@0:    * Returns true if the buffer currently has enough data.
michael@0:    * Returns false if there isn't enough data or if no such track exists.
michael@0:    */
michael@0:   bool HaveEnoughBuffered(TrackID aID);
michael@0:   /**
michael@0:    * Ensures that aSignalRunnable will be dispatched to aSignalThread
michael@0:    * when we don't have enough buffered data in the track (which could be
michael@0:    * immediately). Will dispatch the runnable immediately if the track
michael@0:    * does not exist. No op if a runnable is already present for this track.
michael@0:    */
michael@0:   void DispatchWhenNotEnoughBuffered(TrackID aID,
michael@0:       nsIEventTarget* aSignalThread, nsIRunnable* aSignalRunnable);
michael@0:   /**
michael@0:    * Indicate that a track has ended. Do not do any more API calls
michael@0:    * affecting this track.
michael@0:    * Ignored if the track does not exist.
michael@0:    */
michael@0:   void EndTrack(TrackID aID);
michael@0:   /**
michael@0:    * Indicate that no tracks will be added starting before time aKnownTime.
michael@0:    * aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime.
michael@0:    */
michael@0:   void AdvanceKnownTracksTime(StreamTime aKnownTime);
michael@0:   /**
michael@0:    * Indicate that this stream should enter the "finished" state. All tracks
michael@0:    * must have been ended via EndTrack. The finish time of the stream is
michael@0:    * when all tracks have ended.
michael@0:    */
michael@0:   void FinishWithLockHeld();
michael@0:   void Finish()
michael@0:     {
michael@0:       MutexAutoLock lock(mMutex);
michael@0:       FinishWithLockHeld();
michael@0:     }
michael@0: 
michael@0:   // Overriding allows us to hold the mMutex lock while changing the track enable status
michael@0:   void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) {
michael@0:     MutexAutoLock lock(mMutex);
michael@0:     MediaStream::SetTrackEnabledImpl(aTrackID, aEnabled);
michael@0:   }
michael@0: 
michael@0:   /**
michael@0:    * End all tracks and Finish() this stream.  Used to voluntarily revoke access
michael@0:    * to a LocalMediaStream.
michael@0:    */
michael@0:   void EndAllTrackAndFinish();
michael@0: 
michael@0:   /**
michael@0:    * Note: Only call from Media Graph thread (eg NotifyPull)
michael@0:    *
michael@0:    * Returns amount of time (data) that is currently buffered in the track,
michael@0:    * assuming playout via PlayAudio or via a TrackUnion - note that
michael@0:    * NotifyQueuedTrackChanges() on a SourceMediaStream will occur without
michael@0:    * any "extra" buffering, but NotifyQueued TrackChanges() on a TrackUnion
michael@0:    * will be buffered.
michael@0:    */
michael@0:   TrackTicks GetBufferedTicks(TrackID aID);
michael@0: 
michael@0:   void RegisterForAudioMixing();
michael@0: 
michael@0:   // XXX need a Reset API
michael@0: 
michael@0:   friend class MediaStreamGraphImpl;
michael@0: 
michael@0: protected:
michael@0:   struct ThreadAndRunnable {
michael@0:     void Init(nsIEventTarget* aTarget, nsIRunnable* aRunnable)
michael@0:     {
michael@0:       mTarget = aTarget;
michael@0:       mRunnable = aRunnable;
michael@0:     }
michael@0: 
michael@0:     nsCOMPtr<nsIEventTarget> mTarget;
michael@0:     nsCOMPtr<nsIRunnable> mRunnable;
michael@0:   };
michael@0:   enum TrackCommands {
michael@0:     TRACK_CREATE = MediaStreamListener::TRACK_EVENT_CREATED,
michael@0:     TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED
michael@0:   };
michael@0:   /**
michael@0:    * Data for each track that hasn't ended.
michael@0:    */
michael@0:   struct TrackData {
michael@0:     TrackID mID;
michael@0:     // Sample rate of the input data.
michael@0:     TrackRate mInputRate;
michael@0:     // Sample rate of the output data, always equal to the sample rate of the
michael@0:     // graph.
michael@0:     TrackRate mOutputRate;
michael@0:     // Resampler if the rate of the input track does not match the
michael@0:     // MediaStreamGraph's.
michael@0:     nsAutoRef<SpeexResamplerState> mResampler;
michael@0:     TrackTicks mStart;
michael@0:     // Each time the track updates are flushed to the media graph thread,
michael@0:     // this is cleared.
michael@0:     uint32_t mCommands;
michael@0:     // Each time the track updates are flushed to the media graph thread,
michael@0:     // the segment buffer is emptied.
michael@0:     nsAutoPtr<MediaSegment> mData;
michael@0:     nsTArray<ThreadAndRunnable> mDispatchWhenNotEnough;
michael@0:     bool mHaveEnough;
michael@0:   };
michael@0: 
michael@0:   bool NeedsMixing();
michael@0: 
michael@0:   void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
michael@0: 
michael@0:   TrackData* FindDataForTrack(TrackID aID)
michael@0:   {
michael@0:     for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
michael@0:       if (mUpdateTracks[i].mID == aID) {
michael@0:         return &mUpdateTracks[i];
michael@0:       }
michael@0:     }
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   /**
michael@0:    * Notify direct consumers of new data to one of the stream tracks.
michael@0:    * The data doesn't have to be resampled (though it may be).  This is called
michael@0:    * from AppendToTrack on the thread providing the data, and will call
michael@0:    * the Listeners on this thread.
michael@0:    */
michael@0:   void NotifyDirectConsumers(TrackData *aTrack,
michael@0:                              MediaSegment *aSegment);
michael@0: 
michael@0:   // Media stream graph thread only
michael@0:   MediaStreamListener::Consumption mLastConsumptionState;
michael@0: 
michael@0:   // This must be acquired *before* MediaStreamGraphImpl's lock, if they are
michael@0:   // held together.
michael@0:   Mutex mMutex;
michael@0:   // protected by mMutex
michael@0:   StreamTime mUpdateKnownTracksTime;
michael@0:   nsTArray<TrackData> mUpdateTracks;
michael@0:   nsTArray<nsRefPtr<MediaStreamDirectListener> > mDirectListeners;
michael@0:   bool mPullEnabled;
michael@0:   bool mUpdateFinished;
michael@0:   bool mNeedsMixing;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Represents a connection between a ProcessedMediaStream and one of its
michael@0:  * input streams.
michael@0:  * We make these refcounted so that stream-related messages with MediaInputPort*
michael@0:  * pointers can be sent to the main thread safely.
michael@0:  *
michael@0:  * When a port's source or destination stream dies, the stream's DestroyImpl
michael@0:  * calls MediaInputPort::Disconnect to disconnect the port from
michael@0:  * the source and destination streams.
michael@0:  *
michael@0:  * The lifetimes of MediaInputPort are controlled from the main thread.
michael@0:  * The media graph adds a reference to the port. When a MediaInputPort is no
michael@0:  * longer needed, main-thread code sends a Destroy message for the port and
michael@0:  * clears its reference (the last main-thread reference to the object). When
michael@0:  * the Destroy message is processed on the graph manager thread we disconnect
michael@0:  * the port and drop the graph's reference, destroying the object.
michael@0:  */
michael@0: class MediaInputPort MOZ_FINAL {
michael@0: private:
michael@0:   // Do not call this constructor directly. Instead call aDest->AllocateInputPort.
michael@0:   MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest,
michael@0:                  uint32_t aFlags, uint16_t aInputNumber,
michael@0:                  uint16_t aOutputNumber)
michael@0:     : mSource(aSource)
michael@0:     , mDest(aDest)
michael@0:     , mFlags(aFlags)
michael@0:     , mInputNumber(aInputNumber)
michael@0:     , mOutputNumber(aOutputNumber)
michael@0:     , mGraph(nullptr)
michael@0:   {
michael@0:     MOZ_COUNT_CTOR(MediaInputPort);
michael@0:   }
michael@0: 
michael@0:   // Private destructor, to discourage deletion outside of Release():
michael@0:   ~MediaInputPort()
michael@0:   {
michael@0:     MOZ_COUNT_DTOR(MediaInputPort);
michael@0:   }
michael@0: 
michael@0: public:
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort)
michael@0: 
michael@0:   /**
michael@0:    * The FLAG_BLOCK_INPUT and FLAG_BLOCK_OUTPUT flags can be used to control
michael@0:    * exactly how the blocking statuses of the input and output streams affect
michael@0:    * each other.
michael@0:    */
michael@0:   enum {
michael@0:     // When set, blocking on the output stream forces blocking on the input
michael@0:     // stream.
michael@0:     FLAG_BLOCK_INPUT = 0x01,
michael@0:     // When set, blocking on the input stream forces blocking on the output
michael@0:     // stream.
michael@0:     FLAG_BLOCK_OUTPUT = 0x02
michael@0:   };
michael@0: 
michael@0:   // Called on graph manager thread
michael@0:   // Do not call these from outside MediaStreamGraph.cpp!
michael@0:   void Init();
michael@0:   // Called during message processing to trigger removal of this stream.
michael@0:   void Disconnect();
michael@0: 
michael@0:   // Control API
michael@0:   /**
michael@0:    * Disconnects and destroys the port. The caller must not reference this
michael@0:    * object again.
michael@0:    */
michael@0:   void Destroy();
michael@0: 
michael@0:   // Any thread
michael@0:   MediaStream* GetSource() { return mSource; }
michael@0:   ProcessedMediaStream* GetDestination() { return mDest; }
michael@0: 
michael@0:   uint16_t InputNumber() const { return mInputNumber; }
michael@0:   uint16_t OutputNumber() const { return mOutputNumber; }
michael@0: 
michael@0:   // Call on graph manager thread
michael@0:   struct InputInterval {
michael@0:     GraphTime mStart;
michael@0:     GraphTime mEnd;
michael@0:     bool mInputIsBlocked;
michael@0:   };
michael@0:   // Find the next time interval starting at or after aTime during which
michael@0:   // mDest is not blocked and mSource's blocking status does not change.
michael@0:   InputInterval GetNextInputInterval(GraphTime aTime);
michael@0: 
michael@0:   /**
michael@0:    * Returns the graph that owns this port.
michael@0:    */
michael@0:   MediaStreamGraphImpl* GraphImpl();
michael@0:   MediaStreamGraph* Graph();
michael@0:   /**
michael@0:    * Sets the graph that owns this stream.  Should only be called once.
michael@0:    */
michael@0:   void SetGraphImpl(MediaStreamGraphImpl* aGraph);
michael@0: 
michael@0:   size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
michael@0:   {
michael@0:     size_t amount = 0;
michael@0: 
michael@0:     // Not owned:
michael@0:     // - mSource
michael@0:     // - mDest
michael@0:     // - mGraph
michael@0:     return amount;
michael@0:   }
michael@0: 
michael@0:   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
michael@0:   {
michael@0:     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
michael@0:   }
michael@0: 
michael@0: private:
michael@0:   friend class MediaStreamGraphImpl;
michael@0:   friend class MediaStream;
michael@0:   friend class ProcessedMediaStream;
michael@0:   // Never modified after Init()
michael@0:   MediaStream* mSource;
michael@0:   ProcessedMediaStream* mDest;
michael@0:   uint32_t mFlags;
michael@0:   // The input and output numbers are optional, and are currently only used by
michael@0:   // Web Audio.
michael@0:   const uint16_t mInputNumber;
michael@0:   const uint16_t mOutputNumber;
michael@0: 
michael@0:   // Our media stream graph
michael@0:   MediaStreamGraphImpl* mGraph;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * This stream processes zero or more input streams in parallel to produce
michael@0:  * its output. The details of how the output is produced are handled by
michael@0:  * subclasses overriding the ProcessInput method.
michael@0:  */
michael@0: class ProcessedMediaStream : public MediaStream {
michael@0: public:
michael@0:   ProcessedMediaStream(DOMMediaStream* aWrapper)
michael@0:     : MediaStream(aWrapper), mAutofinish(false), mInCycle(false)
michael@0:   {}
michael@0: 
michael@0:   // Control API.
michael@0:   /**
michael@0:    * Allocates a new input port attached to source aStream.
michael@0:    * This stream can be removed by calling MediaInputPort::Remove().
michael@0:    */
michael@0:   already_AddRefed<MediaInputPort> AllocateInputPort(MediaStream* aStream,
michael@0:                                                      uint32_t aFlags = 0,
michael@0:                                                      uint16_t aInputNumber = 0,
michael@0:                                                      uint16_t aOutputNumber = 0);
michael@0:   /**
michael@0:    * Force this stream into the finished state.
michael@0:    */
michael@0:   void Finish();
michael@0:   /**
michael@0:    * Set the autofinish flag on this stream (defaults to false). When this flag
michael@0:    * is set, and all input streams are in the finished state (including if there
michael@0:    * are no input streams), this stream automatically enters the finished state.
michael@0:    */
michael@0:   void SetAutofinish(bool aAutofinish);
michael@0: 
michael@0:   virtual ProcessedMediaStream* AsProcessedStream() { return this; }
michael@0: 
michael@0:   friend class MediaStreamGraphImpl;
michael@0: 
michael@0:   // Do not call these from outside MediaStreamGraph.cpp!
michael@0:   virtual void AddInput(MediaInputPort* aPort);
michael@0:   virtual void RemoveInput(MediaInputPort* aPort)
michael@0:   {
michael@0:     mInputs.RemoveElement(aPort);
michael@0:   }
michael@0:   bool HasInputPort(MediaInputPort* aPort)
michael@0:   {
michael@0:     return mInputs.Contains(aPort);
michael@0:   }
michael@0:   uint32_t InputPortCount()
michael@0:   {
michael@0:     return mInputs.Length();
michael@0:   }
michael@0:   virtual void DestroyImpl();
michael@0:   /**
michael@0:    * This gets called after we've computed the blocking states for all
michael@0:    * streams (mBlocked is up to date up to mStateComputedTime).
michael@0:    * Also, we've produced output for all streams up to this one. If this stream
michael@0:    * is not in a cycle, then all its source streams have produced data.
michael@0:    * Generate output from aFrom to aTo.
michael@0:    * This will be called on streams that have finished. Most stream types should
michael@0:    * just return immediately if IsFinishedOnGraphThread(), but some may wish to
michael@0:    * update internal state (see AudioNodeStream).
michael@0:    * ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH
michael@0:    * is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e.
michael@0:    * when we've producing the last block of data we need to produce.) Otherwise
michael@0:    * we can get into a situation where we've determined the stream should not
michael@0:    * block before mStateComputedTime, but the stream finishes before
michael@0:    * mStateComputedTime, violating the invariant that finished streams are blocked.
michael@0:    */
michael@0:   enum {
michael@0:     ALLOW_FINISH = 0x01
michael@0:   };
michael@0:   virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0;
michael@0:   void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; }
michael@0: 
michael@0:   /**
michael@0:    * Forward SetTrackEnabled() to the input MediaStream(s) and translate the ID
michael@0:    */
michael@0:   virtual void ForwardTrackEnabled(TrackID aOutputID, bool aEnabled) {};
michael@0: 
michael@0:   bool InCycle() const { return mInCycle; }
michael@0: 
michael@0:   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
michael@0:   {
michael@0:     size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf);
michael@0:     // Not owned:
michael@0:     // - mInputs elements
michael@0:     amount += mInputs.SizeOfExcludingThis(aMallocSizeOf);
michael@0:     return amount;
michael@0:   }
michael@0: 
michael@0:   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
michael@0:   {
michael@0:     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
michael@0:   }
michael@0: 
michael@0: protected:
michael@0:   // This state is all accessed only on the media graph thread.
michael@0: 
michael@0:   // The list of all inputs that are currently enabled or waiting to be enabled.
michael@0:   nsTArray<MediaInputPort*> mInputs;
michael@0:   bool mAutofinish;
michael@0:   // True if and only if this stream is in a cycle.
michael@0:   // Updated by MediaStreamGraphImpl::UpdateStreamOrder.
michael@0:   bool mInCycle;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Initially, at least, we will have a singleton MediaStreamGraph per
michael@0:  * process.  Each OfflineAudioContext object creates its own MediaStreamGraph
michael@0:  * object too.
michael@0:  */
michael@0: class MediaStreamGraph {
michael@0: public:
michael@0:   // We ensure that the graph current time advances in multiples of
michael@0:   // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
michael@0:   // never blocks and has a track with the ideal audio rate will produce audio
michael@0:   // in multiples of the block size.
michael@0: 
michael@0:   // Main thread only
michael@0:   static MediaStreamGraph* GetInstance();
michael@0:   static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
michael@0:   // Idempotent
michael@0:   static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
michael@0: 
michael@0:   // Control API.
michael@0:   /**
michael@0:    * Create a stream that a media decoder (or some other source of
michael@0:    * media data, such as a camera) can write to.
michael@0:    */
michael@0:   SourceMediaStream* CreateSourceStream(DOMMediaStream* aWrapper);
michael@0:   /**
michael@0:    * Create a stream that will form the union of the tracks of its input
michael@0:    * streams.
michael@0:    * A TrackUnionStream contains all the tracks of all its input streams.
michael@0:    * Adding a new input stream makes that stream's tracks immediately appear as new
michael@0:    * tracks starting at the time the input stream was added.
michael@0:    * Removing an input stream makes the output tracks corresponding to the
michael@0:    * removed tracks immediately end.
michael@0:    * For each added track, the track ID of the output track is the track ID
michael@0:    * of the input track or one plus the maximum ID of all previously added
michael@0:    * tracks, whichever is greater.
michael@0:    * TODO at some point we will probably need to add API to select
michael@0:    * particular tracks of each input stream.
michael@0:    */
michael@0:   ProcessedMediaStream* CreateTrackUnionStream(DOMMediaStream* aWrapper);
michael@0:   // Internal AudioNodeStreams can only pass their output to another
michael@0:   // AudioNode, whereas external AudioNodeStreams can pass their output
michael@0:   // to an nsAudioStream for playback.
michael@0:   enum AudioNodeStreamKind { SOURCE_STREAM, INTERNAL_STREAM, EXTERNAL_STREAM };
michael@0:   /**
michael@0:    * Create a stream that will process audio for an AudioNode.
michael@0:    * Takes ownership of aEngine.  aSampleRate is the sampling rate used
michael@0:    * for the stream.  If 0 is passed, the sampling rate of the engine's
michael@0:    * node will get used.
michael@0:    */
michael@0:   AudioNodeStream* CreateAudioNodeStream(AudioNodeEngine* aEngine,
michael@0:                                          AudioNodeStreamKind aKind,
michael@0:                                          TrackRate aSampleRate = 0);
michael@0: 
michael@0:   AudioNodeExternalInputStream*
michael@0:   CreateAudioNodeExternalInputStream(AudioNodeEngine* aEngine,
michael@0:                                      TrackRate aSampleRate = 0);
michael@0: 
michael@0:   bool IsNonRealtime() const;
michael@0:   /**
michael@0:    * Start processing non-realtime for a specific number of ticks.
michael@0:    */
michael@0:   void StartNonRealtimeProcessing(TrackRate aRate, uint32_t aTicksToProcess);
michael@0: 
michael@0:   /**
michael@0:    * Media graph thread only.
michael@0:    * Dispatches a runnable that will run on the main thread after all
michael@0:    * main-thread stream state has been next updated.
michael@0:    * Should only be called during MediaStreamListener callbacks or during
michael@0:    * ProcessedMediaStream::ProcessInput().
michael@0:    */
michael@0:   void DispatchToMainThreadAfterStreamStateUpdate(already_AddRefed<nsIRunnable> aRunnable)
michael@0:   {
michael@0:     *mPendingUpdateRunnables.AppendElement() = aRunnable;
michael@0:   }
michael@0: 
michael@0: protected:
michael@0:   MediaStreamGraph()
michael@0:     : mNextGraphUpdateIndex(1)
michael@0:   {
michael@0:     MOZ_COUNT_CTOR(MediaStreamGraph);
michael@0:   }
michael@0:   virtual ~MediaStreamGraph()
michael@0:   {
michael@0:     MOZ_COUNT_DTOR(MediaStreamGraph);
michael@0:   }
michael@0: 
michael@0:   // Media graph thread only
michael@0:   nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
michael@0: 
michael@0:   // Main thread only
michael@0:   // The number of updates we have sent to the media graph thread + 1.
michael@0:   // We start this at 1 just to ensure that 0 is usable as a special value.
michael@0:   int64_t mNextGraphUpdateIndex;
michael@0: };
michael@0: 
michael@0: }
michael@0: 
michael@0: #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */