The Tor Browser: content/media/MediaStreamGraph.h@97036ab72558 (annotated)

content/media/MediaStreamGraph.h@97036ab72558 (annotated)

content/media/MediaStreamGraph.h

Tue, 06 Jan 2015 21:39:09 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Tue, 06 Jan 2015 21:39:09 +0100
branch: TOR_BUG_9701
changeset 8: 97036ab72558
permissions: -rw-r--r--

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 #ifndef MOZILLA_MEDIASTREAMGRAPH_H_
 #define MOZILLA_MEDIASTREAMGRAPH_H_
 #include "mozilla/Mutex.h"
 #include "mozilla/LinkedList.h"
 #include "AudioStream.h"
 #include "nsTArray.h"
 #include "nsIRunnable.h"
 #include "StreamBuffer.h"
 #include "TimeVarying.h"
 #include "VideoFrameContainer.h"
 #include "VideoSegment.h"
 #include "MainThreadUtils.h"
 #include "nsAutoRef.h"
 #include "speex/speex_resampler.h"
 #include "AudioMixer.h"
 #include "mozilla/dom/AudioChannelBinding.h"
 class nsIRunnable;
 template <>
 class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState>
 {
   public:
   static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); }
 };
 namespace mozilla {
 class DOMMediaStream;
 #ifdef PR_LOGGING
 extern PRLogModuleInfo* gMediaStreamGraphLog;
 #endif
 /**
  * Microseconds relative to the start of the graph timeline.
  */
 typedef int64_t GraphTime;
 const GraphTime GRAPH_TIME_MAX = MEDIA_TIME_MAX;
 /*
  * MediaStreamGraph is a framework for synchronized audio/video processing
  * and playback. It is designed to be used by other browser components such as
  * HTML media elements, media capture APIs, real-time media streaming APIs,
  * multitrack media APIs, and advanced audio APIs.
  *
  * The MediaStreamGraph uses a dedicated thread to process media --- the media
  * graph thread. This ensures that we can process media through the graph
  * without blocking on main-thread activity. The media graph is only modified
  * on the media graph thread, to ensure graph changes can be processed without
  * interfering with media processing. All interaction with the media graph
  * thread is done with message passing.
  *
  * APIs that modify the graph or its properties are described as "control APIs".
  * These APIs are asynchronous; they queue graph changes internally and
  * those changes are processed all-at-once by the MediaStreamGraph. The
  * MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState
  * to ensure that graph changes from a single event loop task are always
  * processed all together. Control APIs should only be used on the main thread,
  * currently; we may be able to relax that later.
  *
  * To allow precise synchronization of times in the control API, the
  * MediaStreamGraph maintains a "media timeline". Control APIs that take or
  * return times use that timeline. Those times never advance during
  * an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime().
  *
  * Media decoding, audio processing and media playback use thread-safe APIs to
  * the media graph to ensure they can continue while the main thread is blocked.
  *
  * When the graph is changed, we may need to throw out buffered data and
  * reprocess it. This is triggered automatically by the MediaStreamGraph.
  */
 class MediaStreamGraph;
 /**
  * This is a base class for media graph thread listener callbacks.
  * Override methods to be notified of audio or video data or changes in stream
  * state.
  *
  * This can be used by stream recorders or network connections that receive
  * stream input. It could also be used for debugging.
  *
  * All notification methods are called from the media graph thread. Overriders
  * of these methods are responsible for all synchronization. Beware!
  * These methods are called without the media graph monitor held, so
  * reentry into media graph methods is possible, although very much discouraged!
  * You should do something non-blocking and non-reentrant (e.g. dispatch an
  * event to some thread) and return.
  * The listener is not allowed to add/remove any listeners from the stream.
  *
  * When a listener is first attached, we guarantee to send a NotifyBlockingChanged
  * callback to notify of the initial blocking state. Also, if a listener is
  * attached to a stream that has already finished, we'll call NotifyFinished.
  */
 class MediaStreamListener {
 protected:
   // Protected destructor, to discourage deletion outside of Release():
   virtual ~MediaStreamListener() {}
 public:
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener)
   enum Consumption {
     CONSUMED,
     NOT_CONSUMED
   };
   /**
    * Notify that the stream is hooked up and we'd like to start or stop receiving
    * data on it. Only fires on SourceMediaStreams.
    * The initial state is assumed to be NOT_CONSUMED.
    */
   virtual void NotifyConsumptionChanged(MediaStreamGraph* aGraph, Consumption aConsuming) {}
   /**
    * When a SourceMediaStream has pulling enabled, and the MediaStreamGraph
    * control loop is ready to pull, this gets called. A NotifyPull implementation
    * is allowed to call the SourceMediaStream methods that alter track
    * data. It is not allowed to make other MediaStream API calls, including
    * calls to add or remove MediaStreamListeners. It is not allowed to block
    * for any length of time.
    * aDesiredTime is the stream time we would like to get data up to. Data
    * beyond this point will not be played until NotifyPull runs again, so there's
    * not much point in providing it. Note that if the stream is blocked for
    * some reason, then data before aDesiredTime may not be played immediately.
    */
   virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) {}
   enum Blocking {
     BLOCKED,
     UNBLOCKED
   };
   /**
    * Notify that the blocking status of the stream changed. The initial state
    * is assumed to be BLOCKED.
    */
   virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) {}
   /**
    * Notify that the stream has data in each track
    * for the stream's current time. Once this state becomes true, it will
    * always be true since we block stream time from progressing to times where
    * there isn't data in each track.
    */
   virtual void NotifyHasCurrentData(MediaStreamGraph* aGraph) {}
   /**
    * Notify that the stream output is advancing. aCurrentTime is the graph's
    * current time. MediaStream::GraphTimeToStreamTime can be used to get the
    * stream time.
    */
   virtual void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) {}
   /**
    * Notify that the stream finished.
    */
   virtual void NotifyFinished(MediaStreamGraph* aGraph) {}
   /**
    * Notify that your listener has been removed, either due to RemoveListener(),
    * or due to the stream being destroyed.  You will get no further notifications.
    */
   virtual void NotifyRemoved(MediaStreamGraph* aGraph) {}
   enum {
     TRACK_EVENT_CREATED = 0x01,
     TRACK_EVENT_ENDED = 0x02
   };
   /**
    * Notify that changes to one of the stream tracks have been queued.
    * aTrackEvents can be any combination of TRACK_EVENT_CREATED and
    * TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
    * at aTrackOffset (relative to the start of the stream).
    */
   virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
                                         TrackRate aTrackRate,
                                         TrackTicks aTrackOffset,
                                         uint32_t aTrackEvents,
                                         const MediaSegment& aQueuedMedia) {}
 };
 /**
  * This is a base class for media graph thread listener direct callbacks
  * from within AppendToTrack().  Note that your regular listener will
  * still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
  * you must be careful to ignore them if AddDirectListener was successful.
  */
 class MediaStreamDirectListener : public MediaStreamListener
 {
 public:
   virtual ~MediaStreamDirectListener() {}
   /*
    * This will be called on any MediaStreamDirectListener added to a
    * a SourceMediaStream when AppendToTrack() is called.  The MediaSegment
    * will be the RawSegment (unresampled) if available in AppendToTrack().
    * Note that NotifyQueuedTrackChanges() calls will also still occur.
    */
   virtual void NotifyRealtimeData(MediaStreamGraph* aGraph, TrackID aID,
                                   TrackRate aTrackRate,
                                   TrackTicks aTrackOffset,
                                   uint32_t aTrackEvents,
                                   const MediaSegment& aMedia) {}
 };
 /**
  * This is a base class for main-thread listener callbacks.
  * This callback is invoked on the main thread when the main-thread-visible
  * state of a stream has changed.
  *
  * These methods are called with the media graph monitor held, so
  * reentry into general media graph methods is not possible.
  * You should do something non-blocking and non-reentrant (e.g. dispatch an
  * event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
  * would be a good choice.
  * The listener is allowed to synchronously remove itself from the stream, but
  * not add or remove any other listeners.
  */
 class MainThreadMediaStreamListener {
 public:
   virtual void NotifyMainThreadStateChanged() = 0;
 };
 /**
  * Helper struct used to keep track of memory usage by AudioNodes.
  */
 struct AudioNodeSizes
 {
   size_t mDomNode;
   size_t mStream;
   size_t mEngine;
   nsCString mNodeType;
 };
 class MediaStreamGraphImpl;
 class SourceMediaStream;
 class ProcessedMediaStream;
 class MediaInputPort;
 class AudioNodeEngine;
 class AudioNodeExternalInputStream;
 class AudioNodeStream;
 struct AudioChunk;
 /**
  * A stream of synchronized audio and video data. All (not blocked) streams
  * progress at the same rate --- "real time". Streams cannot seek. The only
  * operation readers can perform on a stream is to read the next data.
  *
  * Consumers of a stream can be reading from it at different offsets, but that
  * should only happen due to the order in which consumers are being run.
  * Those offsets must not diverge in the long term, otherwise we would require
  * unbounded buffering.
  *
  * Streams can be in a "blocked" state. While blocked, a stream does not
  * produce data. A stream can be explicitly blocked via the control API,
  * or implicitly blocked by whatever's generating it (e.g. an underrun in the
  * source resource), or implicitly blocked because something consuming it
  * blocks, or implicitly because it has finished.
  *
  * A stream can be in a "finished" state. "Finished" streams are permanently
  * blocked.
  *
  * Transitions into and out of the "blocked" and "finished" states are managed
  * by the MediaStreamGraph on the media graph thread.
  *
  * We buffer media data ahead of the consumers' reading offsets. It is possible
  * to have buffered data but still be blocked.
  *
  * Any stream can have its audio and video playing when requested. The media
  * stream graph plays audio by constructing audio output streams as necessary.
  * Video is played by setting video frames into an VideoFrameContainer at the right
  * time. To ensure video plays in sync with audio, make sure that the same
  * stream is playing both the audio and video.
  *
  * The data in a stream is managed by StreamBuffer. It consists of a set of
  * tracks of various types that can start and end over time.
  *
  * Streams are explicitly managed. The client creates them via
  * MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling
  * Destroy() when no longer needed (actual destruction will be deferred).
  * The actual object is owned by the MediaStreamGraph. The basic idea is that
  * main thread objects will keep Streams alive as long as necessary (using the
  * cycle collector to clean up whenever needed).
  *
  * We make them refcounted only so that stream-related messages with MediaStream*
  * pointers can be sent to the main thread safely.
  *
  * The lifetimes of MediaStreams are controlled from the main thread.
  * For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM
  * wrapper; the DOM wrappers own their associated MediaStreams. When a DOM
  * wrapper is destroyed, it sends a Destroy message for the associated
  * MediaStream and clears its reference (the last main-thread reference to
  * the object). When the Destroy message is processed on the graph
  * manager thread we immediately release the affected objects (disentangling them
  * from other objects as necessary).
  *
  * This could cause problems for media processing if a MediaStream is
  * destroyed while a downstream MediaStream is still using it. Therefore
  * the DOM wrappers must keep upstream MediaStreams alive as long as they
  * could be being used in the media graph.
  *
  * At any time, however, a set of MediaStream wrappers could be
  * collected via cycle collection. Destroy messages will be sent
  * for those objects in arbitrary order and the MediaStreamGraph has to be able
  * to handle this.
  */
 class MediaStream : public mozilla::LinkedListElement<MediaStream> {
 public:
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
   MediaStream(DOMMediaStream* aWrapper);
 protected:
   // Protected destructor, to discourage deletion outside of Release():
   virtual ~MediaStream()
   {
     MOZ_COUNT_DTOR(MediaStream);
     NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
     NS_ASSERTION(mMainThreadListeners.IsEmpty(),
                  "All main thread listeners should have been removed");
   }
 public:
   /**
    * Returns the graph that owns this stream.
    */
   MediaStreamGraphImpl* GraphImpl();
   MediaStreamGraph* Graph();
   /**
    * Sets the graph that owns this stream.  Should only be called once.
    */
   void SetGraphImpl(MediaStreamGraphImpl* aGraph);
   void SetGraphImpl(MediaStreamGraph* aGraph);
   // Control API.
   // Since a stream can be played multiple ways, we need to combine independent
   // volume settings. The aKey parameter is used to keep volume settings
   // separate. Since the stream is always playing the same contents, only
   // a single audio output stream is used; the volumes are combined.
   // Currently only the first enabled audio track is played.
   // XXX change this so all enabled audio tracks are mixed and played.
   virtual void AddAudioOutput(void* aKey);
   virtual void SetAudioOutputVolume(void* aKey, float aVolume);
   virtual void RemoveAudioOutput(void* aKey);
   // Since a stream can be played multiple ways, we need to be able to
   // play to multiple VideoFrameContainers.
   // Only the first enabled video track is played.
   virtual void AddVideoOutput(VideoFrameContainer* aContainer);
   virtual void RemoveVideoOutput(VideoFrameContainer* aContainer);
   // Explicitly block. Useful for example if a media element is pausing
   // and we need to stop its stream emitting its buffered data.
   virtual void ChangeExplicitBlockerCount(int32_t aDelta);
   // Events will be dispatched by calling methods of aListener.
   virtual void AddListener(MediaStreamListener* aListener);
   virtual void RemoveListener(MediaStreamListener* aListener);
   // A disabled track has video replaced by black, and audio replaced by
   // silence.
   void SetTrackEnabled(TrackID aTrackID, bool aEnabled);
   // Events will be dispatched by calling methods of aListener. It is the
   // responsibility of the caller to remove aListener before it is destroyed.
   void AddMainThreadListener(MainThreadMediaStreamListener* aListener)
   {
     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
     mMainThreadListeners.AppendElement(aListener);
   }
   // It's safe to call this even if aListener is not currently a listener;
   // the call will be ignored.
   void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener)
   {
     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
     mMainThreadListeners.RemoveElement(aListener);
   }
   /**
    * Ensure a runnable will run on the main thread after running all pending
    * updates that were sent from the graph thread or will be sent before the
    * graph thread receives the next graph update.
    *
    * If the graph has been shut down or destroyed, then the runnable will be
    * dispatched to the event queue immediately.  If the graph is non-realtime
    * and has not started, then the runnable will be run
    * synchronously/immediately.  (There are no pending updates in these
    * situations.)
    *
    * Main thread only.
    */
   void RunAfterPendingUpdates(nsRefPtr<nsIRunnable> aRunnable);
   // Signal that the client is done with this MediaStream. It will be deleted later.
   virtual void Destroy();
   // Returns the main-thread's view of how much data has been processed by
   // this stream.
   StreamTime GetCurrentTime()
   {
     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
     return mMainThreadCurrentTime;
   }
   // Return the main thread's view of whether this stream has finished.
   bool IsFinished()
   {
     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
     return mMainThreadFinished;
   }
   bool IsDestroyed()
   {
     NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
     return mMainThreadDestroyed;
   }
   friend class MediaStreamGraphImpl;
   friend class MediaInputPort;
   friend class AudioNodeExternalInputStream;
   virtual SourceMediaStream* AsSourceStream() { return nullptr; }
   virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
   virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
   // media graph thread only
   void Init();
   // These Impl methods perform the core functionality of the control methods
   // above, on the media graph thread.
   /**
    * Stop all stream activity and disconnect it from all inputs and outputs.
    * This must be idempotent.
    */
   virtual void DestroyImpl();
   StreamTime GetBufferEnd() { return mBuffer.GetEnd(); }
 #ifdef DEBUG
   void DumpTrackInfo() { return mBuffer.DumpTrackInfo(); }
 #endif
   void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
   void AddAudioOutputImpl(void* aKey)
   {
     mAudioOutputs.AppendElement(AudioOutput(aKey));
   }
   void RemoveAudioOutputImpl(void* aKey);
   void AddVideoOutputImpl(already_AddRefed<VideoFrameContainer> aContainer)
   {
     *mVideoOutputs.AppendElement() = aContainer;
   }
   void RemoveVideoOutputImpl(VideoFrameContainer* aContainer)
   {
     mVideoOutputs.RemoveElement(aContainer);
   }
   void ChangeExplicitBlockerCountImpl(GraphTime aTime, int32_t aDelta)
   {
     mExplicitBlockerCount.SetAtAndAfter(aTime, mExplicitBlockerCount.GetAt(aTime) + aDelta);
   }
   void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
   void RemoveListenerImpl(MediaStreamListener* aListener);
   void RemoveAllListenersImpl();
   void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled);
   /**
    * Returns true when this stream requires the contents of its inputs even if
    * its own outputs are not being consumed. This is used to signal inputs to
    * this stream that they are being consumed; when they're not being consumed,
    * we make some optimizations.
    */
   virtual bool IsIntrinsicallyConsumed() const
   {
     return !mAudioOutputs.IsEmpty() || !mVideoOutputs.IsEmpty();
   }
   void AddConsumer(MediaInputPort* aPort)
   {
     mConsumers.AppendElement(aPort);
   }
   void RemoveConsumer(MediaInputPort* aPort)
   {
     mConsumers.RemoveElement(aPort);
   }
   uint32_t ConsumerCount()
   {
     return mConsumers.Length();
   }
   const StreamBuffer& GetStreamBuffer() { return mBuffer; }
   GraphTime GetStreamBufferStartTime() { return mBufferStartTime; }
   /**
    * Convert graph time to stream time. aTime must be <= mStateComputedTime
    * to ensure we know exactly how much time this stream will be blocked during
    * the interval.
    */
   StreamTime GraphTimeToStreamTime(GraphTime aTime);
   /**
    * Convert graph time to stream time. aTime can be > mStateComputedTime,
    * in which case we optimistically assume the stream will not be blocked
    * after mStateComputedTime.
    */
   StreamTime GraphTimeToStreamTimeOptimistic(GraphTime aTime);
   /**
    * Convert stream time to graph time. The result can be > mStateComputedTime,
    * in which case we did the conversion optimistically assuming the stream
    * will not be blocked after mStateComputedTime.
    */
   GraphTime StreamTimeToGraphTime(StreamTime aTime);
   bool IsFinishedOnGraphThread() { return mFinished; }
   void FinishOnGraphThread();
   /**
    * Identify which graph update index we are currently processing.
    */
   int64_t GetProcessingGraphUpdateIndex();
   bool HasCurrentData() { return mHasCurrentData; }
   StreamBuffer::Track* EnsureTrack(TrackID aTrack, TrackRate aSampleRate);
   void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr);
   DOMMediaStream* GetWrapper()
   {
     NS_ASSERTION(NS_IsMainThread(), "Only use DOMMediaStream on main thread");
     return mWrapper;
   }
   // Return true if the main thread needs to observe updates from this stream.
   virtual bool MainThreadNeedsUpdates() const
   {
     return true;
   }
   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
   void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; }
 protected:
   virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
   {
     mBufferStartTime += aBlockedTime;
     mGraphUpdateIndices.InsertTimeAtStart(aBlockedTime);
     mGraphUpdateIndices.AdvanceCurrentTime(aCurrentTime);
     mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime);
     mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime);
   }
   // This state is all initialized on the main thread but
   // otherwise modified only on the media graph thread.
   // Buffered data. The start of the buffer corresponds to mBufferStartTime.
   // Conceptually the buffer contains everything this stream has ever played,
   // but we forget some prefix of the buffered data to bound the space usage.
   StreamBuffer mBuffer;
   // The time when the buffered data could be considered to have started playing.
   // This increases over time to account for time the stream was blocked before
   // mCurrentTime.
   GraphTime mBufferStartTime;
   // Client-set volume of this stream
   struct AudioOutput {
     AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
     void* mKey;
     float mVolume;
   };
   nsTArray<AudioOutput> mAudioOutputs;
   nsTArray<nsRefPtr<VideoFrameContainer> > mVideoOutputs;
   // We record the last played video frame to avoid redundant setting
   // of the current video frame.
   VideoFrame mLastPlayedVideoFrame;
   // The number of times this stream has been explicitly blocked by the control
   // API, minus the number of times it has been explicitly unblocked.
   TimeVarying<GraphTime,uint32_t,0> mExplicitBlockerCount;
   nsTArray<nsRefPtr<MediaStreamListener> > mListeners;
   nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
   nsTArray<TrackID> mDisabledTrackIDs;
   // Precomputed blocking status (over GraphTime).
   // This is only valid between the graph's mCurrentTime and
   // mStateComputedTime. The stream is considered to have
   // not been blocked before mCurrentTime (its mBufferStartTime is increased
   // as necessary to account for that time instead) --- this avoids us having to
   // record the entire history of the stream's blocking-ness in mBlocked.
   TimeVarying<GraphTime,bool,5> mBlocked;
   // Maps graph time to the graph update that affected this stream at that time
   TimeVarying<GraphTime,int64_t,0> mGraphUpdateIndices;
   // MediaInputPorts to which this is connected
   nsTArray<MediaInputPort*> mConsumers;
   // Where audio output is going. There is one AudioOutputStream per
   // audio track.
   struct AudioOutputStream {
     // When we started audio playback for this track.
     // Add mStream->GetPosition() to find the current audio playback position.
     GraphTime mAudioPlaybackStartTime;
     // Amount of time that we've wanted to play silence because of the stream
     // blocking.
     MediaTime mBlockedAudioTime;
     // Last tick written to the audio output.
     TrackTicks mLastTickWritten;
     RefPtr<AudioStream> mStream;
     TrackID mTrackID;
     size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
     {
       size_t amount = 0;
       amount += mStream->SizeOfIncludingThis(aMallocSizeOf);
       return amount;
     }
   };
   nsTArray<AudioOutputStream> mAudioOutputStreams;
   /**
    * When true, this means the stream will be finished once all
    * buffered data has been consumed.
    */
   bool mFinished;
   /**
    * When true, mFinished is true and we've played all the data in this stream
    * and fired NotifyFinished notifications.
    */
   bool mNotifiedFinished;
   /**
    * When true, the last NotifyBlockingChanged delivered to the listeners
    * indicated that the stream is blocked.
    */
   bool mNotifiedBlocked;
   /**
    * True if some data can be present by this stream if/when it's unblocked.
    * Set by the stream itself on the MediaStreamGraph thread. Only changes
    * from false to true once a stream has data, since we won't
    * unblock it until there's more data.
    */
   bool mHasCurrentData;
   /**
    * True if mHasCurrentData is true and we've notified listeners.
    */
   bool mNotifiedHasCurrentData;
   // Temporary data for ordering streams by dependency graph
   bool mHasBeenOrdered;
   bool mIsOnOrderingStack;
   // True if the stream is being consumed (i.e. has track data being played,
   // or is feeding into some stream that is being consumed).
   bool mIsConsumed;
   // Temporary data for computing blocking status of streams
   // True if we've added this stream to the set of streams we're computing
   // blocking for.
   bool mInBlockingSet;
   // True if this stream should be blocked in this phase.
   bool mBlockInThisPhase;
   // This state is only used on the main thread.
   DOMMediaStream* mWrapper;
   // Main-thread views of state
   StreamTime mMainThreadCurrentTime;
   bool mMainThreadFinished;
   bool mMainThreadDestroyed;
   // Our media stream graph.  null if destroyed on the graph thread.
   MediaStreamGraphImpl* mGraph;
   dom::AudioChannel mAudioChannelType;
 };
 /**
  * This is a stream into which a decoder can write audio and video.
  *
  * Audio and video can be written on any thread, but you probably want to
  * always write from the same thread to avoid unexpected interleavings.
  */
 class SourceMediaStream : public MediaStream {
 public:
   SourceMediaStream(DOMMediaStream* aWrapper) :
     MediaStream(aWrapper),
     mLastConsumptionState(MediaStreamListener::NOT_CONSUMED),
     mMutex("mozilla::media::SourceMediaStream"),
     mUpdateKnownTracksTime(0),
     mPullEnabled(false),
     mUpdateFinished(false)
   {}
   virtual SourceMediaStream* AsSourceStream() { return this; }
   // Media graph thread only
   virtual void DestroyImpl();
   // Call these on any thread.
   /**
    * Enable or disable pulling. When pulling is enabled, NotifyPull
    * gets called on MediaStreamListeners for this stream during the
    * MediaStreamGraph control loop. Pulling is initially disabled.
    * Due to unavoidable race conditions, after a call to SetPullEnabled(false)
    * it is still possible for a NotifyPull to occur.
    */
   void SetPullEnabled(bool aEnabled);
   void AddDirectListener(MediaStreamDirectListener* aListener);
   void RemoveDirectListener(MediaStreamDirectListener* aListener);
   /**
    * Add a new track to the stream starting at the given base time (which
    * must be greater than or equal to the last time passed to
    * AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should
    * contain data starting after aStart.
    */
   void AddTrack(TrackID aID, TrackRate aRate, TrackTicks aStart,
                 MediaSegment* aSegment);
   /**
    * Append media data to a track. Ownership of aSegment remains with the caller,
    * but aSegment is emptied.
    * Returns false if the data was not appended because no such track exists
    * or the stream was already finished.
    */
   bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
   /**
    * Returns true if the buffer currently has enough data.
    * Returns false if there isn't enough data or if no such track exists.
    */
   bool HaveEnoughBuffered(TrackID aID);
   /**
    * Ensures that aSignalRunnable will be dispatched to aSignalThread
    * when we don't have enough buffered data in the track (which could be
    * immediately). Will dispatch the runnable immediately if the track
    * does not exist. No op if a runnable is already present for this track.
    */
   void DispatchWhenNotEnoughBuffered(TrackID aID,
       nsIEventTarget* aSignalThread, nsIRunnable* aSignalRunnable);
   /**
    * Indicate that a track has ended. Do not do any more API calls
    * affecting this track.
    * Ignored if the track does not exist.
    */
   void EndTrack(TrackID aID);
   /**
    * Indicate that no tracks will be added starting before time aKnownTime.
    * aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime.
    */
   void AdvanceKnownTracksTime(StreamTime aKnownTime);
   /**
    * Indicate that this stream should enter the "finished" state. All tracks
    * must have been ended via EndTrack. The finish time of the stream is
    * when all tracks have ended.
    */
   void FinishWithLockHeld();
   void Finish()
     {
       MutexAutoLock lock(mMutex);
       FinishWithLockHeld();
     }
   // Overriding allows us to hold the mMutex lock while changing the track enable status
   void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) {
     MutexAutoLock lock(mMutex);
     MediaStream::SetTrackEnabledImpl(aTrackID, aEnabled);
   }
   /**
    * End all tracks and Finish() this stream.  Used to voluntarily revoke access
    * to a LocalMediaStream.
    */
   void EndAllTrackAndFinish();
   /**
    * Note: Only call from Media Graph thread (eg NotifyPull)
    *
    * Returns amount of time (data) that is currently buffered in the track,
    * assuming playout via PlayAudio or via a TrackUnion - note that
    * NotifyQueuedTrackChanges() on a SourceMediaStream will occur without
    * any "extra" buffering, but NotifyQueued TrackChanges() on a TrackUnion
    * will be buffered.
    */
   TrackTicks GetBufferedTicks(TrackID aID);
   void RegisterForAudioMixing();
   // XXX need a Reset API
   friend class MediaStreamGraphImpl;
 protected:
   struct ThreadAndRunnable {
     void Init(nsIEventTarget* aTarget, nsIRunnable* aRunnable)
     {
       mTarget = aTarget;
       mRunnable = aRunnable;
     }
     nsCOMPtr<nsIEventTarget> mTarget;
     nsCOMPtr<nsIRunnable> mRunnable;
   };
   enum TrackCommands {
     TRACK_CREATE = MediaStreamListener::TRACK_EVENT_CREATED,
     TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED
   };
   /**
    * Data for each track that hasn't ended.
    */
   struct TrackData {
     TrackID mID;
     // Sample rate of the input data.
     TrackRate mInputRate;
     // Sample rate of the output data, always equal to the sample rate of the
     // graph.
     TrackRate mOutputRate;
     // Resampler if the rate of the input track does not match the
     // MediaStreamGraph's.
     nsAutoRef<SpeexResamplerState> mResampler;
     TrackTicks mStart;
     // Each time the track updates are flushed to the media graph thread,
     // this is cleared.
     uint32_t mCommands;
     // Each time the track updates are flushed to the media graph thread,
     // the segment buffer is emptied.
     nsAutoPtr<MediaSegment> mData;
     nsTArray<ThreadAndRunnable> mDispatchWhenNotEnough;
     bool mHaveEnough;
   };
   bool NeedsMixing();
   void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
   TrackData* FindDataForTrack(TrackID aID)
   {
     for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
       if (mUpdateTracks[i].mID == aID) {
         return &mUpdateTracks[i];
       }
     }
     return nullptr;
   }
   /**
    * Notify direct consumers of new data to one of the stream tracks.
    * The data doesn't have to be resampled (though it may be).  This is called
    * from AppendToTrack on the thread providing the data, and will call
    * the Listeners on this thread.
    */
   void NotifyDirectConsumers(TrackData *aTrack,
                              MediaSegment *aSegment);
   // Media stream graph thread only
   MediaStreamListener::Consumption mLastConsumptionState;
   // This must be acquired *before* MediaStreamGraphImpl's lock, if they are
   // held together.
   Mutex mMutex;
   // protected by mMutex
   StreamTime mUpdateKnownTracksTime;
   nsTArray<TrackData> mUpdateTracks;
   nsTArray<nsRefPtr<MediaStreamDirectListener> > mDirectListeners;
   bool mPullEnabled;
   bool mUpdateFinished;
   bool mNeedsMixing;
 };
 /**
  * Represents a connection between a ProcessedMediaStream and one of its
  * input streams.
  * We make these refcounted so that stream-related messages with MediaInputPort*
  * pointers can be sent to the main thread safely.
  *
  * When a port's source or destination stream dies, the stream's DestroyImpl
  * calls MediaInputPort::Disconnect to disconnect the port from
  * the source and destination streams.
  *
  * The lifetimes of MediaInputPort are controlled from the main thread.
  * The media graph adds a reference to the port. When a MediaInputPort is no
  * longer needed, main-thread code sends a Destroy message for the port and
  * clears its reference (the last main-thread reference to the object). When
  * the Destroy message is processed on the graph manager thread we disconnect
  * the port and drop the graph's reference, destroying the object.
  */
 class MediaInputPort MOZ_FINAL {
 private:
   // Do not call this constructor directly. Instead call aDest->AllocateInputPort.
   MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest,
                  uint32_t aFlags, uint16_t aInputNumber,
                  uint16_t aOutputNumber)
     : mSource(aSource)
     , mDest(aDest)
     , mFlags(aFlags)
     , mInputNumber(aInputNumber)
     , mOutputNumber(aOutputNumber)
     , mGraph(nullptr)
   {
     MOZ_COUNT_CTOR(MediaInputPort);
   }
   // Private destructor, to discourage deletion outside of Release():
   ~MediaInputPort()
   {
     MOZ_COUNT_DTOR(MediaInputPort);
   }
 public:
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort)
   /**
    * The FLAG_BLOCK_INPUT and FLAG_BLOCK_OUTPUT flags can be used to control
    * exactly how the blocking statuses of the input and output streams affect
    * each other.
    */
   enum {
     // When set, blocking on the output stream forces blocking on the input
     // stream.
     FLAG_BLOCK_INPUT = 0x01,
     // When set, blocking on the input stream forces blocking on the output
     // stream.
     FLAG_BLOCK_OUTPUT = 0x02
   };
   // Called on graph manager thread
   // Do not call these from outside MediaStreamGraph.cpp!
   void Init();
   // Called during message processing to trigger removal of this stream.
   void Disconnect();
   // Control API
   /**
    * Disconnects and destroys the port. The caller must not reference this
    * object again.
    */
   void Destroy();
   // Any thread
   MediaStream* GetSource() { return mSource; }
   ProcessedMediaStream* GetDestination() { return mDest; }
   uint16_t InputNumber() const { return mInputNumber; }
   uint16_t OutputNumber() const { return mOutputNumber; }
   // Call on graph manager thread
   struct InputInterval {
     GraphTime mStart;
     GraphTime mEnd;
     bool mInputIsBlocked;
   };
   // Find the next time interval starting at or after aTime during which
   // mDest is not blocked and mSource's blocking status does not change.
   InputInterval GetNextInputInterval(GraphTime aTime);
   /**
    * Returns the graph that owns this port.
    */
   MediaStreamGraphImpl* GraphImpl();
   MediaStreamGraph* Graph();
   /**
    * Sets the graph that owns this stream.  Should only be called once.
    */
   void SetGraphImpl(MediaStreamGraphImpl* aGraph);
   size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
   {
     size_t amount = 0;
     // Not owned:
     // - mSource
     // - mDest
     // - mGraph
     return amount;
   }
   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
   {
     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
   }
 private:
   friend class MediaStreamGraphImpl;
   friend class MediaStream;
   friend class ProcessedMediaStream;
   // Never modified after Init()
   MediaStream* mSource;
   ProcessedMediaStream* mDest;
   uint32_t mFlags;
   // The input and output numbers are optional, and are currently only used by
   // Web Audio.
   const uint16_t mInputNumber;
   const uint16_t mOutputNumber;
   // Our media stream graph
   MediaStreamGraphImpl* mGraph;
 };
 /**
  * This stream processes zero or more input streams in parallel to produce
  * its output. The details of how the output is produced are handled by
  * subclasses overriding the ProcessInput method.
  */
 class ProcessedMediaStream : public MediaStream {
 public:
   ProcessedMediaStream(DOMMediaStream* aWrapper)
     : MediaStream(aWrapper), mAutofinish(false), mInCycle(false)
   {}
   // Control API.
   /**
    * Allocates a new input port attached to source aStream.
    * This stream can be removed by calling MediaInputPort::Remove().
    */
   already_AddRefed<MediaInputPort> AllocateInputPort(MediaStream* aStream,
                                                      uint32_t aFlags = 0,
                                                      uint16_t aInputNumber = 0,
                                                      uint16_t aOutputNumber = 0);
   /**
    * Force this stream into the finished state.
    */
   void Finish();
   /**
    * Set the autofinish flag on this stream (defaults to false). When this flag
    * is set, and all input streams are in the finished state (including if there
    * are no input streams), this stream automatically enters the finished state.
    */
   void SetAutofinish(bool aAutofinish);
   virtual ProcessedMediaStream* AsProcessedStream() { return this; }
   friend class MediaStreamGraphImpl;
   // Do not call these from outside MediaStreamGraph.cpp!
   virtual void AddInput(MediaInputPort* aPort);
   virtual void RemoveInput(MediaInputPort* aPort)
   {
     mInputs.RemoveElement(aPort);
   }
   bool HasInputPort(MediaInputPort* aPort)
   {
     return mInputs.Contains(aPort);
   }
   uint32_t InputPortCount()
   {
     return mInputs.Length();
   }
   virtual void DestroyImpl();
   /**
    * This gets called after we've computed the blocking states for all
    * streams (mBlocked is up to date up to mStateComputedTime).
    * Also, we've produced output for all streams up to this one. If this stream
    * is not in a cycle, then all its source streams have produced data.
    * Generate output from aFrom to aTo.
    * This will be called on streams that have finished. Most stream types should
    * just return immediately if IsFinishedOnGraphThread(), but some may wish to
    * update internal state (see AudioNodeStream).
    * ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH
    * is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e.
    * when we've producing the last block of data we need to produce.) Otherwise
    * we can get into a situation where we've determined the stream should not
    * block before mStateComputedTime, but the stream finishes before
    * mStateComputedTime, violating the invariant that finished streams are blocked.
    */
   enum {
     ALLOW_FINISH = 0x01
   };
   virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0;
   void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; }
   /**
    * Forward SetTrackEnabled() to the input MediaStream(s) and translate the ID
    */
   virtual void ForwardTrackEnabled(TrackID aOutputID, bool aEnabled) {};
   bool InCycle() const { return mInCycle; }
   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
   {
     size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf);
     // Not owned:
     // - mInputs elements
     amount += mInputs.SizeOfExcludingThis(aMallocSizeOf);
     return amount;
   }
   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
   {
     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
   }
 protected:
   // This state is all accessed only on the media graph thread.
   // The list of all inputs that are currently enabled or waiting to be enabled.
   nsTArray<MediaInputPort*> mInputs;
   bool mAutofinish;
   // True if and only if this stream is in a cycle.
   // Updated by MediaStreamGraphImpl::UpdateStreamOrder.
   bool mInCycle;
 };
 /**
  * Initially, at least, we will have a singleton MediaStreamGraph per
  * process.  Each OfflineAudioContext object creates its own MediaStreamGraph
  * object too.
  */
 class MediaStreamGraph {
 public:
   // We ensure that the graph current time advances in multiples of
   // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
   // never blocks and has a track with the ideal audio rate will produce audio
   // in multiples of the block size.
   // Main thread only
   static MediaStreamGraph* GetInstance();
   static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
   // Idempotent
   static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
   // Control API.
   /**
    * Create a stream that a media decoder (or some other source of
    * media data, such as a camera) can write to.
    */
   SourceMediaStream* CreateSourceStream(DOMMediaStream* aWrapper);
   /**
    * Create a stream that will form the union of the tracks of its input
    * streams.
    * A TrackUnionStream contains all the tracks of all its input streams.
    * Adding a new input stream makes that stream's tracks immediately appear as new
    * tracks starting at the time the input stream was added.
    * Removing an input stream makes the output tracks corresponding to the
    * removed tracks immediately end.
    * For each added track, the track ID of the output track is the track ID
    * of the input track or one plus the maximum ID of all previously added
    * tracks, whichever is greater.
    * TODO at some point we will probably need to add API to select
    * particular tracks of each input stream.
    */
   ProcessedMediaStream* CreateTrackUnionStream(DOMMediaStream* aWrapper);
   // Internal AudioNodeStreams can only pass their output to another
   // AudioNode, whereas external AudioNodeStreams can pass their output
   // to an nsAudioStream for playback.
   enum AudioNodeStreamKind { SOURCE_STREAM, INTERNAL_STREAM, EXTERNAL_STREAM };
   /**
    * Create a stream that will process audio for an AudioNode.
    * Takes ownership of aEngine.  aSampleRate is the sampling rate used
    * for the stream.  If 0 is passed, the sampling rate of the engine's
    * node will get used.
    */
   AudioNodeStream* CreateAudioNodeStream(AudioNodeEngine* aEngine,
                                          AudioNodeStreamKind aKind,
                                          TrackRate aSampleRate = 0);
   AudioNodeExternalInputStream*
   CreateAudioNodeExternalInputStream(AudioNodeEngine* aEngine,
                                      TrackRate aSampleRate = 0);
   bool IsNonRealtime() const;
   /**
    * Start processing non-realtime for a specific number of ticks.
    */
   void StartNonRealtimeProcessing(TrackRate aRate, uint32_t aTicksToProcess);
   /**
    * Media graph thread only.
    * Dispatches a runnable that will run on the main thread after all
    * main-thread stream state has been next updated.
    * Should only be called during MediaStreamListener callbacks or during
    * ProcessedMediaStream::ProcessInput().
    */
   void DispatchToMainThreadAfterStreamStateUpdate(already_AddRefed<nsIRunnable> aRunnable)
   {
     *mPendingUpdateRunnables.AppendElement() = aRunnable;
   }
 protected:
   MediaStreamGraph()
     : mNextGraphUpdateIndex(1)
   {
     MOZ_COUNT_CTOR(MediaStreamGraph);
   }
   virtual ~MediaStreamGraph()
   {
     MOZ_COUNT_DTOR(MediaStreamGraph);
   }
   // Media graph thread only
   nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
   // Main thread only
   // The number of updates we have sent to the media graph thread + 1.
   // We start this at 1 just to ensure that 0 is usable as a special value.
   int64_t mNextGraphUpdateIndex;
 };
 }
 #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */

The Tor Browser / annotate

content/media/MediaStreamGraph.h@97036ab72558 (annotated)

content/media/MediaStreamGraph.h